2023-07-22 12:35:06,938 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@13c9d689 2023-07-22 12:35:06,951 main DEBUG Took 0.010763 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-07-22 12:35:06,951 main DEBUG PluginManager 'Core' found 129 plugins 2023-07-22 12:35:06,951 main DEBUG PluginManager 'Level' found 0 plugins 2023-07-22 12:35:06,952 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-07-22 12:35:06,953 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:35:06,963 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-07-22 12:35:06,978 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:35:06,980 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:35:06,981 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:35:06,981 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:35:06,982 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:35:06,982 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:35:06,983 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:35:06,983 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:35:06,984 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:35:06,984 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:35:06,984 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:35:06,985 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:35:06,985 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:35:06,986 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:35:06,986 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:35:06,987 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:35:06,987 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:35:06,988 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:35:06,988 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:35:06,989 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:35:06,989 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:35:06,989 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:35:06,990 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:35:06,991 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:35:06,991 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:35:06,991 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-07-22 12:35:06,994 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:35:06,997 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-07-22 12:35:06,999 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-07-22 12:35:06,999 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-07-22 12:35:07,000 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-07-22 12:35:07,000 main DEBUG PluginManager 'Converter' found 47 plugins 2023-07-22 12:35:07,012 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-07-22 12:35:07,015 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-07-22 12:35:07,018 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-07-22 12:35:07,019 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-07-22 12:35:07,019 main DEBUG createAppenders(={Console}) 2023-07-22 12:35:07,020 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@13c9d689 initialized 2023-07-22 12:35:07,020 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@13c9d689 2023-07-22 12:35:07,020 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@13c9d689 OK. 2023-07-22 12:35:07,021 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-07-22 12:35:07,021 main DEBUG OutputStream closed 2023-07-22 12:35:07,021 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-07-22 12:35:07,022 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-07-22 12:35:07,022 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@15c43bd9 OK 2023-07-22 12:35:07,064 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-07-22 12:35:07,066 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-07-22 12:35:07,067 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-07-22 12:35:07,068 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-07-22 12:35:07,069 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-07-22 12:35:07,069 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-07-22 12:35:07,069 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-07-22 12:35:07,070 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-07-22 12:35:07,070 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-07-22 12:35:07,070 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-07-22 12:35:07,070 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-07-22 12:35:07,071 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-07-22 12:35:07,071 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-07-22 12:35:07,071 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-07-22 12:35:07,072 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-07-22 12:35:07,072 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-07-22 12:35:07,072 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-07-22 12:35:07,073 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-07-22 12:35:07,076 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-07-22 12:35:07,076 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@1b6e1eff) with optional ClassLoader: null 2023-07-22 12:35:07,076 main DEBUG Shutdown hook enabled. Registering a new one. 2023-07-22 12:35:07,077 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@1b6e1eff] started OK. 2023-07-22T12:35:07,107 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestRemoteRestore timeout: 13 mins 2023-07-22 12:35:07,110 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-07-22 12:35:07,111 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-07-22T12:35:07,402 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93 2023-07-22T12:35:07,407 DEBUG [Time-limited test {}] impl.BackupManager(127): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-07-22T12:35:07,409 DEBUG [Time-limited test {}] impl.BackupManager(157): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:35:07,410 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-07-22T12:35:07,413 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/cluster_81664056-bd3e-8e52-f121-db92c3ef923e, deleteOnExit=true 2023-07-22T12:35:07,413 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-07-22T12:35:07,414 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/test.cache.data in system properties and HBase conf 2023-07-22T12:35:07,414 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/hadoop.tmp.dir in system properties and HBase conf 2023-07-22T12:35:07,415 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/hadoop.log.dir in system properties and HBase conf 2023-07-22T12:35:07,415 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/mapreduce.cluster.local.dir in system properties and HBase conf 2023-07-22T12:35:07,415 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-07-22T12:35:07,415 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-07-22T12:35:07,509 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-07-22T12:35:07,981 DEBUG [Time-limited test {}] fs.HFileSystem(308): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-07-22T12:35:07,987 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-07-22T12:35:07,988 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-07-22T12:35:07,988 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-07-22T12:35:07,988 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-07-22T12:35:07,989 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-07-22T12:35:07,989 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-07-22T12:35:07,989 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-07-22T12:35:07,990 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/dfs.journalnode.edits.dir in system properties and HBase conf 2023-07-22T12:35:07,990 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-07-22T12:35:07,990 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/nfs.dump.dir in system properties and HBase conf 2023-07-22T12:35:07,990 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/java.io.tmpdir in system properties and HBase conf 2023-07-22T12:35:07,991 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/dfs.journalnode.edits.dir in system properties and HBase conf 2023-07-22T12:35:07,991 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-07-22T12:35:07,991 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-07-22T12:35:08,412 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-07-22T12:35:08,417 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-07-22T12:35:08,700 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-07-22T12:35:08,862 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@5623674c via org.mortbay.log.Slf4jLog 2023-07-22T12:35:08,877 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:35:08,914 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:35:08,948 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/java.io.tmpdir/Jetty_localhost_33421_hdfs____.sf8cw3/webapp 2023-07-22T12:35:09,078 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:33421 2023-07-22T12:35:09,097 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-07-22T12:35:09,097 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-07-22T12:35:09,594 WARN [Listener at localhost/38685 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-07-22T12:35:09,657 WARN [Listener at localhost/38685 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-07-22T12:35:09,675 WARN [Listener at localhost/38685 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:35:09,681 INFO [Listener at localhost/38685 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:35:09,686 INFO [Listener at localhost/38685 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/java.io.tmpdir/Jetty_localhost_46567_datanode____30aeor/webapp 2023-07-22T12:35:09,787 INFO [Listener at localhost/38685 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:46567 2023-07-22T12:35:10,055 WARN [Listener at localhost/39493 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-07-22T12:35:10,329 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-07-22T12:35:10,515 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x9eb20668faa24d98: Processing first storage report for DS-c76e90c8-39fa-4f11-ba17-53eb00da3429 from datanode ebfe7662-6a3f-4444-8812-84efc237aad3 2023-07-22T12:35:10,516 DEBUG [Listener at localhost/39493 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93 2023-07-22T12:35:10,517 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x9eb20668faa24d98: from storage DS-c76e90c8-39fa-4f11-ba17-53eb00da3429 node DatanodeRegistration(127.0.0.1:38553, datanodeUuid=ebfe7662-6a3f-4444-8812-84efc237aad3, infoPort=41867, infoSecurePort=0, ipcPort=39493, storageInfo=lv=-57;cid=testClusterID;nsid=1266225153;c=1690029308490), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2023-07-22T12:35:10,518 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x9eb20668faa24d98: Processing first storage report for DS-6ff2313b-dafc-411f-bd06-ac38c3079f40 from datanode ebfe7662-6a3f-4444-8812-84efc237aad3 2023-07-22T12:35:10,518 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x9eb20668faa24d98: from storage DS-6ff2313b-dafc-411f-bd06-ac38c3079f40 node DatanodeRegistration(127.0.0.1:38553, datanodeUuid=ebfe7662-6a3f-4444-8812-84efc237aad3, infoPort=41867, infoSecurePort=0, ipcPort=39493, storageInfo=lv=-57;cid=testClusterID;nsid=1266225153;c=1690029308490), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-07-22T12:35:10,605 INFO [Listener at localhost/39493 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/cluster_81664056-bd3e-8e52-f121-db92c3ef923e/zookeeper_0, clientPort=54609, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/cluster_81664056-bd3e-8e52-f121-db92c3ef923e/zookeeper_0/version-2, dataDirSize=424 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/cluster_81664056-bd3e-8e52-f121-db92c3ef923e/zookeeper_0/version-2, dataLogSize=424 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, serverId=0 2023-07-22T12:35:10,624 INFO [Listener at localhost/39493 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=54609 2023-07-22T12:35:10,633 INFO [Listener at localhost/39493 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:35:10,636 INFO [Listener at localhost/39493 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:35:11,309 INFO [Listener at localhost/39493 {}] util.FSUtils(461): Created version file at hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f with version=8 2023-07-22T12:35:11,309 INFO [Listener at localhost/39493 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/hbase-staging 2023-07-22T12:35:11,326 DEBUG [Listener at localhost/39493 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-07-22T12:35:11,326 DEBUG [Listener at localhost/39493 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-07-22T12:35:11,326 DEBUG [Listener at localhost/39493 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-07-22T12:35:11,326 DEBUG [Listener at localhost/39493 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-07-22T12:35:11,547 DEBUG [Listener at localhost/39493 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-07-22T12:35:11,696 INFO [Listener at localhost/39493 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-07-22T12:35:12,356 INFO [Listener at localhost/39493 {}] client.ConnectionUtils(127): master/jenkins-hbase3:0 server-side Connection retries=45 2023-07-22T12:35:12,402 INFO [Listener at localhost/39493 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-07-22T12:35:12,403 INFO [Listener at localhost/39493 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-07-22T12:35:12,403 INFO [Listener at localhost/39493 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-07-22T12:35:12,403 INFO [Listener at localhost/39493 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-07-22T12:35:12,403 INFO [Listener at localhost/39493 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-07-22T12:35:12,550 INFO [Listener at localhost/39493 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-07-22T12:35:12,653 DEBUG [Listener at localhost/39493 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-07-22T12:35:12,659 INFO [Listener at localhost/39493 {}] ipc.NettyRpcServer(177): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-07-22T12:35:12,716 DEBUG [Listener at localhost/39493 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 19984 (auto-detected) 2023-07-22T12:35:12,717 DEBUG [Listener at localhost/39493 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:1c:ff:fe:e2:13:ea (auto-detected) 2023-07-22T12:35:12,754 INFO [Listener at localhost/39493 {}] ipc.NettyRpcServer(144): Bind to /172.31.12.81:46137 2023-07-22T12:35:12,774 INFO [Listener at localhost/39493 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:35:12,777 INFO [Listener at localhost/39493 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:35:12,795 INFO [Listener at localhost/39493 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:46137 connecting to ZooKeeper ensemble=127.0.0.1:54609 2023-07-22T12:35:12,841 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:461370x0, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-07-22T12:35:12,845 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:46137-0x100006222d70000 connected 2023-07-22T12:35:12,879 DEBUG [Listener at localhost/39493 {}] zookeeper.ZKUtil(113): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-07-22T12:35:12,883 DEBUG [Listener at localhost/39493 {}] zookeeper.ZKUtil(113): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-07-22T12:35:12,887 DEBUG [Listener at localhost/39493 {}] zookeeper.ZKUtil(113): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-07-22T12:35:12,899 DEBUG [Listener at localhost/39493 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46137 2023-07-22T12:35:12,900 DEBUG [Listener at localhost/39493 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46137 2023-07-22T12:35:12,900 DEBUG [Listener at localhost/39493 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46137 2023-07-22T12:35:12,901 DEBUG [Listener at localhost/39493 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46137 2023-07-22T12:35:12,901 DEBUG [Listener at localhost/39493 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46137 2023-07-22T12:35:12,955 INFO [Listener at localhost/39493 {}] log.Log(170): Logging initialized @6691ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2023-07-22T12:35:13,079 INFO [Listener at localhost/39493 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-07-22T12:35:13,080 INFO [Listener at localhost/39493 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-07-22T12:35:13,081 INFO [Listener at localhost/39493 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-07-22T12:35:13,083 INFO [Listener at localhost/39493 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-07-22T12:35:13,083 INFO [Listener at localhost/39493 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-07-22T12:35:13,084 INFO [Listener at localhost/39493 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-07-22T12:35:13,087 INFO [Listener at localhost/39493 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-07-22T12:35:13,223 INFO [Listener at localhost/39493 {}] http.HttpServer(1219): Jetty bound to port 37443 2023-07-22T12:35:13,225 INFO [Listener at localhost/39493 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-07-22T12:35:13,256 INFO [Listener at localhost/39493 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:35:13,259 INFO [Listener at localhost/39493 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@65657a93{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/hadoop.log.dir/,AVAILABLE} 2023-07-22T12:35:13,260 INFO [Listener at localhost/39493 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:35:13,260 INFO [Listener at localhost/39493 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@1997ec4d{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-07-22T12:35:13,321 INFO [Listener at localhost/39493 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-07-22T12:35:13,335 INFO [Listener at localhost/39493 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-07-22T12:35:13,335 INFO [Listener at localhost/39493 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-07-22T12:35:13,337 INFO [Listener at localhost/39493 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2023-07-22T12:35:13,344 INFO [Listener at localhost/39493 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:35:14,515 INFO [Listener at localhost/39493 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@42323d37{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-07-22T12:35:14,526 INFO [Listener at localhost/39493 {}] server.AbstractConnector(333): Started ServerConnector@33d7be8d{HTTP/1.1, (http/1.1)}{0.0.0.0:37443} 2023-07-22T12:35:14,526 INFO [Listener at localhost/39493 {}] server.Server(415): Started @8263ms 2023-07-22T12:35:14,530 INFO [Listener at localhost/39493 {}] master.HMaster(484): hbase.rootdir=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f, hbase.cluster.distributed=false 2023-07-22T12:35:14,594 INFO [Listener at localhost/39493 {}] client.ConnectionUtils(127): regionserver/jenkins-hbase3:0 server-side Connection retries=45 2023-07-22T12:35:14,594 INFO [Listener at localhost/39493 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-07-22T12:35:14,594 INFO [Listener at localhost/39493 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-07-22T12:35:14,595 INFO [Listener at localhost/39493 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-07-22T12:35:14,595 INFO [Listener at localhost/39493 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-07-22T12:35:14,595 INFO [Listener at localhost/39493 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-07-22T12:35:14,603 INFO [Listener at localhost/39493 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-07-22T12:35:14,605 INFO [Listener at localhost/39493 {}] ipc.NettyRpcServer(177): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-07-22T12:35:14,606 INFO [Listener at localhost/39493 {}] ipc.NettyRpcServer(144): Bind to /172.31.12.81:38147 2023-07-22T12:35:14,609 INFO [Listener at localhost/39493 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-07-22T12:35:14,617 DEBUG [Listener at localhost/39493 {}] mob.MobFileCache(120): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-07-22T12:35:14,618 INFO [Listener at localhost/39493 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:35:14,621 INFO [Listener at localhost/39493 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:35:14,623 INFO [Listener at localhost/39493 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:38147 connecting to ZooKeeper ensemble=127.0.0.1:54609 2023-07-22T12:35:14,627 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): regionserver:381470x0, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-07-22T12:35:14,629 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:38147-0x100006222d70001 connected 2023-07-22T12:35:14,629 DEBUG [Listener at localhost/39493 {}] zookeeper.ZKUtil(113): regionserver:381470x0, quorum=127.0.0.1:54609, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-07-22T12:35:14,630 DEBUG [Listener at localhost/39493 {}] zookeeper.ZKUtil(113): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-07-22T12:35:14,631 DEBUG [Listener at localhost/39493 {}] zookeeper.ZKUtil(113): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-07-22T12:35:14,632 DEBUG [Listener at localhost/39493 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38147 2023-07-22T12:35:14,632 DEBUG [Listener at localhost/39493 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38147 2023-07-22T12:35:14,632 DEBUG [Listener at localhost/39493 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38147 2023-07-22T12:35:14,633 DEBUG [Listener at localhost/39493 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38147 2023-07-22T12:35:14,633 DEBUG [Listener at localhost/39493 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38147 2023-07-22T12:35:14,637 INFO [Listener at localhost/39493 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-07-22T12:35:14,638 INFO [Listener at localhost/39493 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-07-22T12:35:14,638 INFO [Listener at localhost/39493 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-07-22T12:35:14,639 INFO [Listener at localhost/39493 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-07-22T12:35:14,639 INFO [Listener at localhost/39493 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-07-22T12:35:14,639 INFO [Listener at localhost/39493 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-07-22T12:35:14,639 INFO [Listener at localhost/39493 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-07-22T12:35:14,642 INFO [Listener at localhost/39493 {}] http.HttpServer(1219): Jetty bound to port 34185 2023-07-22T12:35:14,642 INFO [Listener at localhost/39493 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-07-22T12:35:14,649 INFO [Listener at localhost/39493 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:35:14,650 INFO [Listener at localhost/39493 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@11566dea{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/hadoop.log.dir/,AVAILABLE} 2023-07-22T12:35:14,650 INFO [Listener at localhost/39493 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:35:14,650 INFO [Listener at localhost/39493 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@132f9d7f{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-07-22T12:35:14,663 INFO [Listener at localhost/39493 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-07-22T12:35:14,664 INFO [Listener at localhost/39493 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-07-22T12:35:14,664 INFO [Listener at localhost/39493 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-07-22T12:35:14,664 INFO [Listener at localhost/39493 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2023-07-22T12:35:14,665 INFO [Listener at localhost/39493 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:35:14,670 INFO [Listener at localhost/39493 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@771961df{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-07-22T12:35:14,671 INFO [Listener at localhost/39493 {}] server.AbstractConnector(333): Started ServerConnector@1167861e{HTTP/1.1, (http/1.1)}{0.0.0.0:34185} 2023-07-22T12:35:14,671 INFO [Listener at localhost/39493 {}] server.Server(415): Started @8407ms 2023-07-22T12:35:14,676 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-07-22T12:35:14,679 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@427cb4e2{HTTP/1.1, (http/1.1)}{0.0.0.0:39109} 2023-07-22T12:35:14,679 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(415): Started @8416ms 2023-07-22T12:35:14,679 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2387): Adding backup master ZNode /1/backup-masters/jenkins-hbase3.apache.org,46137,1690029311532 2023-07-22T12:35:14,693 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-07-22T12:35:14,693 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-07-22T12:35:14,696 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/jenkins-hbase3.apache.org,46137,1690029311532 2023-07-22T12:35:14,701 DEBUG [M:0;jenkins-hbase3:46137 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:46137 2023-07-22T12:35:14,716 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-07-22T12:35:14,716 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-07-22T12:35:14,716 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:35:14,717 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:35:14,717 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Set watcher on existing znode=/1/master 2023-07-22T12:35:14,719 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/jenkins-hbase3.apache.org,46137,1690029311532 from backup master directory 2023-07-22T12:35:14,719 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Set watcher on existing znode=/1/master 2023-07-22T12:35:14,723 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/jenkins-hbase3.apache.org,46137,1690029311532 2023-07-22T12:35:14,723 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-07-22T12:35:14,723 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-07-22T12:35:14,724 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-07-22T12:35:14,724 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,46137,1690029311532 2023-07-22T12:35:14,727 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-07-22T12:35:14,729 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-07-22T12:35:14,815 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(610): Created cluster ID file at hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/hbase.id with ID: 9940c85f-8584-47a1-b52b-06d1a3ec3dd4 2023-07-22T12:35:14,860 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:35:14,878 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:35:14,878 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:35:14,932 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:35:14,935 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-07-22T12:35:14,964 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(270): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-07-22T12:35:14,964 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(276): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-07-22T12:35:14,966 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(285): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:283) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:310) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:950) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:573) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:570) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-07-22T12:35:14,981 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:950) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:573) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:570) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-07-22T12:35:14,983 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-07-22T12:35:15,063 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/MasterData/data/master/store 2023-07-22T12:35:15,490 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-07-22T12:35:15,491 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:35:15,491 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-07-22T12:35:15,491 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:35:15,492 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:35:15,492 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-07-22T12:35:15,492 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:35:15,492 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:35:15,492 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-07-22T12:35:15,494 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/MasterData/data/master/store/.initializing 2023-07-22T12:35:15,494 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/MasterData/WALs/jenkins-hbase3.apache.org,46137,1690029311532 2023-07-22T12:35:15,505 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-07-22T12:35:15,521 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C46137%2C1690029311532, suffix=, logDir=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/MasterData/WALs/jenkins-hbase3.apache.org,46137,1690029311532, archiveDir=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/MasterData/oldWALs, maxLogs=10 2023-07-22T12:35:15,549 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/MasterData/WALs/jenkins-hbase3.apache.org,46137,1690029311532/jenkins-hbase3.apache.org%2C46137%2C1690029311532.1690029315528, exclude list is [], retry=0 2023-07-22T12:35:15,575 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38553,DS-c76e90c8-39fa-4f11-ba17-53eb00da3429,DISK] 2023-07-22T12:35:15,582 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:346) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:120) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:430) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:424) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:120) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:482) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:477) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:489) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:397) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-07-22T12:35:15,661 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/MasterData/WALs/jenkins-hbase3.apache.org,46137,1690029311532/jenkins-hbase3.apache.org%2C46137%2C1690029311532.1690029315528 2023-07-22T12:35:15,661 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38553,DS-c76e90c8-39fa-4f11-ba17-53eb00da3429,DISK]] 2023-07-22T12:35:15,662 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:35:15,663 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:35:15,669 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:35:15,670 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:35:15,743 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:35:15,780 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-07-22T12:35:15,788 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:15,791 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:35:15,792 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:35:15,795 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-07-22T12:35:15,796 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:15,797 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:35:15,797 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:35:15,800 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-07-22T12:35:15,800 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:15,802 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:35:15,802 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:35:15,805 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-07-22T12:35:15,805 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:15,806 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:35:15,812 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:35:15,813 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:35:15,833 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-07-22T12:35:15,838 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:35:15,842 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:35:15,843 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10143650560, jitterRate=-0.05529892444610596}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-07-22T12:35:15,850 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-07-22T12:35:15,852 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-07-22T12:35:15,888 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@da0f70f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:35:15,946 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(859): No meta location available on zookeeper, skip migrating... 2023-07-22T12:35:15,967 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-07-22T12:35:15,968 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(564): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-07-22T12:35:15,971 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-07-22T12:35:15,972 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(584): Recovered RegionProcedureStore lease in 1 msec 2023-07-22T12:35:15,979 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(598): Loaded RegionProcedureStore in 6 msec 2023-07-22T12:35:15,979 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-07-22T12:35:16,015 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-07-22T12:35:16,034 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2023-07-22T12:35:16,036 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/balancer already deleted, retry=false 2023-07-22T12:35:16,039 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(150): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-07-22T12:35:16,041 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2023-07-22T12:35:16,042 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/normalizer already deleted, retry=false 2023-07-22T12:35:16,046 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-07-22T12:35:16,052 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2023-07-22T12:35:16,055 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/split already deleted, retry=false 2023-07-22T12:35:16,056 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2023-07-22T12:35:16,057 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/merge already deleted, retry=false 2023-07-22T12:35:16,074 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2023-07-22T12:35:16,075 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/snapshot-cleanup already deleted, retry=false 2023-07-22T12:35:16,080 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-07-22T12:35:16,080 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-07-22T12:35:16,080 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:35:16,080 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:35:16,081 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(803): Active/primary master=jenkins-hbase3.apache.org,46137,1690029311532, sessionid=0x100006222d70000, setting cluster-up flag (Was=false) 2023-07-22T12:35:16,095 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-07-22T12:35:16,105 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:35:16,105 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:35:16,111 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2023-07-22T12:35:16,113 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,46137,1690029311532 2023-07-22T12:35:16,116 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:35:16,116 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:35:16,127 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2023-07-22T12:35:16,128 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,46137,1690029311532 2023-07-22T12:35:16,133 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:35:16,133 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:35:16,139 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2023-07-22T12:35:16,140 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,46137,1690029311532 2023-07-22T12:35:16,185 DEBUG [RS:0;jenkins-hbase3:38147 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:38147 2023-07-22T12:35:16,186 INFO [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(997): ClusterId : 9940c85f-8584-47a1-b52b-06d1a3ec3dd4 2023-07-22T12:35:16,188 INFO [RS:0;jenkins-hbase3:38147 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-07-22T12:35:16,191 DEBUG [RS:0;jenkins-hbase3:38147 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-07-22T12:35:16,197 DEBUG [RS:0;jenkins-hbase3:38147 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-07-22T12:35:16,197 DEBUG [RS:0;jenkins-hbase3:38147 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-07-22T12:35:16,203 DEBUG [RS:0;jenkins-hbase3:38147 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-07-22T12:35:16,203 DEBUG [RS:0;jenkins-hbase3:38147 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-07-22T12:35:16,206 DEBUG [RS:0;jenkins-hbase3:38147 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-07-22T12:35:16,207 DEBUG [RS:0;jenkins-hbase3:38147 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ff2f246, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:35:16,209 DEBUG [RS:0;jenkins-hbase3:38147 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6784f930, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-07-22T12:35:16,214 INFO [RS:0;jenkins-hbase3:38147 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-07-22T12:35:16,214 INFO [RS:0;jenkins-hbase3:38147 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-07-22T12:35:16,214 DEBUG [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-07-22T12:35:16,217 INFO [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(3023): reportForDuty to master=jenkins-hbase3.apache.org,46137,1690029311532 with isa=jenkins-hbase3.apache.org/172.31.12.81:38147, startcode=1690029314593 2023-07-22T12:35:16,236 DEBUG [RS:0;jenkins-hbase3:38147 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-07-22T12:35:16,320 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:53035, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-07-22T12:35:16,332 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46137 {}] ipc.MetricsHBaseServer(144): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3180) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:588) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17393) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-07-22T12:35:16,337 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-07-22T12:35:16,350 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-07-22T12:35:16,358 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-07-22T12:35:16,369 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,46137,1690029311532 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-07-22T12:35:16,375 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-07-22T12:35:16,375 DEBUG [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(3042): Master is not running yet 2023-07-22T12:35:16,376 WARN [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(1088): reportForDuty failed; sleeping 100 ms and then retrying. 2023-07-22T12:35:16,376 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-07-22T12:35:16,376 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-07-22T12:35:16,376 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-07-22T12:35:16,376 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-07-22T12:35:16,376 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:35:16,376 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-07-22T12:35:16,376 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:35:16,383 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1690029346383 2023-07-22T12:35:16,387 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-07-22T12:35:16,392 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-07-22T12:35:16,392 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-07-22T12:35:16,393 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-07-22T12:35:16,400 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:16,400 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-07-22T12:35:16,406 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-07-22T12:35:16,407 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-07-22T12:35:16,408 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-07-22T12:35:16,408 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-07-22T12:35:16,409 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-07-22T12:35:16,410 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-07-22T12:35:16,412 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-07-22T12:35:16,415 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-07-22T12:35:16,416 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-07-22T12:35:16,427 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x12c3adfa to 127.0.0.1:54609 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:35:16,443 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12a85513, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:35:16,444 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-07-22T12:35:16,447 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-07-22T12:35:16,447 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-07-22T12:35:16,449 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1690029316449,5,FailOnTimeoutGroup] 2023-07-22T12:35:16,449 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1690029316449,5,FailOnTimeoutGroup] 2023-07-22T12:35:16,449 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-07-22T12:35:16,450 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1634): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-07-22T12:35:16,452 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-07-22T12:35:16,452 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-07-22T12:35:16,477 INFO [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(3023): reportForDuty to master=jenkins-hbase3.apache.org,46137,1690029311532 with isa=jenkins-hbase3.apache.org/172.31.12.81:38147, startcode=1690029314593 2023-07-22T12:35:16,482 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46137 {}] master.ServerManager(403): Registering regionserver=jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:16,495 DEBUG [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f 2023-07-22T12:35:16,495 DEBUG [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:38685 2023-07-22T12:35:16,496 DEBUG [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=37443 2023-07-22T12:35:16,507 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-07-22T12:35:16,508 DEBUG [RS:0;jenkins-hbase3:38147 {}] zookeeper.ZKUtil(111): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Set watcher on existing znode=/1/rs/jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:16,508 WARN [RS:0;jenkins-hbase3:38147 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-07-22T12:35:16,508 INFO [RS:0;jenkins-hbase3:38147 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-07-22T12:35:16,508 DEBUG [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/WALs/jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:16,511 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,38147,1690029314593] 2023-07-22T12:35:16,527 DEBUG [RS:0;jenkins-hbase3:38147 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-07-22T12:35:16,537 INFO [RS:0;jenkins-hbase3:38147 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-07-22T12:35:16,552 INFO [RS:0;jenkins-hbase3:38147 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-07-22T12:35:16,557 INFO [RS:0;jenkins-hbase3:38147 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-07-22T12:35:16,557 INFO [RS:0;jenkins-hbase3:38147 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-07-22T12:35:16,560 INFO [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-07-22T12:35:16,569 INFO [RS:0;jenkins-hbase3:38147 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-07-22T12:35:16,570 DEBUG [RS:0;jenkins-hbase3:38147 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:35:16,570 DEBUG [RS:0;jenkins-hbase3:38147 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:35:16,570 DEBUG [RS:0;jenkins-hbase3:38147 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:35:16,570 DEBUG [RS:0;jenkins-hbase3:38147 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:35:16,570 DEBUG [RS:0;jenkins-hbase3:38147 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:35:16,570 DEBUG [RS:0;jenkins-hbase3:38147 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-07-22T12:35:16,570 DEBUG [RS:0;jenkins-hbase3:38147 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:35:16,570 DEBUG [RS:0;jenkins-hbase3:38147 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:35:16,571 DEBUG [RS:0;jenkins-hbase3:38147 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:35:16,571 DEBUG [RS:0;jenkins-hbase3:38147 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:35:16,571 DEBUG [RS:0;jenkins-hbase3:38147 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:35:16,571 DEBUG [RS:0;jenkins-hbase3:38147 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-07-22T12:35:16,572 INFO [RS:0;jenkins-hbase3:38147 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-07-22T12:35:16,572 INFO [RS:0;jenkins-hbase3:38147 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-07-22T12:35:16,572 INFO [RS:0;jenkins-hbase3:38147 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-07-22T12:35:16,572 INFO [RS:0;jenkins-hbase3:38147 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-07-22T12:35:16,573 INFO [RS:0;jenkins-hbase3:38147 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,38147,1690029314593-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-07-22T12:35:16,593 INFO [RS:0;jenkins-hbase3:38147 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-07-22T12:35:16,596 INFO [RS:0;jenkins-hbase3:38147 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,38147,1690029314593-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-07-22T12:35:16,613 INFO [RS:0;jenkins-hbase3:38147 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,38147,1690029314593 started 2023-07-22T12:35:16,613 INFO [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,38147,1690029314593, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:38147, sessionid=0x100006222d70001 2023-07-22T12:35:16,614 DEBUG [RS:0;jenkins-hbase3:38147 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-07-22T12:35:16,614 DEBUG [RS:0;jenkins-hbase3:38147 {}] flush.RegionServerFlushTableProcedureManager(106): Start region server flush procedure manager jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:16,614 DEBUG [RS:0;jenkins-hbase3:38147 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,38147,1690029314593' 2023-07-22T12:35:16,614 DEBUG [RS:0;jenkins-hbase3:38147 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2023-07-22T12:35:16,615 DEBUG [RS:0;jenkins-hbase3:38147 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2023-07-22T12:35:16,616 DEBUG [RS:0;jenkins-hbase3:38147 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-07-22T12:35:16,616 DEBUG [RS:0;jenkins-hbase3:38147 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-07-22T12:35:16,616 DEBUG [RS:0;jenkins-hbase3:38147 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,38147,1690029314593' 2023-07-22T12:35:16,616 DEBUG [RS:0;jenkins-hbase3:38147 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:35:16,616 DEBUG [RS:0;jenkins-hbase3:38147 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:35:16,617 INFO [RS:0;jenkins-hbase3:38147 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-07-22T12:35:16,617 DEBUG [RS:0;jenkins-hbase3:38147 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-07-22T12:35:16,617 DEBUG [RS:0;jenkins-hbase3:38147 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-07-22T12:35:16,617 DEBUG [RS:0;jenkins-hbase3:38147 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:16,617 DEBUG [RS:0;jenkins-hbase3:38147 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,38147,1690029314593' 2023-07-22T12:35:16,617 DEBUG [RS:0;jenkins-hbase3:38147 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2023-07-22T12:35:16,618 DEBUG [RS:0;jenkins-hbase3:38147 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2023-07-22T12:35:16,618 DEBUG [RS:0;jenkins-hbase3:38147 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-07-22T12:35:16,618 INFO [RS:0;jenkins-hbase3:38147 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-07-22T12:35:16,618 INFO [RS:0;jenkins-hbase3:38147 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-07-22T12:35:16,729 INFO [RS:0;jenkins-hbase3:38147 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-07-22T12:35:16,732 INFO [RS:0;jenkins-hbase3:38147 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C38147%2C1690029314593, suffix=, logDir=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/WALs/jenkins-hbase3.apache.org,38147,1690029314593, archiveDir=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/oldWALs, maxLogs=32 2023-07-22T12:35:16,744 DEBUG [RS:0;jenkins-hbase3:38147 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/WALs/jenkins-hbase3.apache.org,38147,1690029314593/jenkins-hbase3.apache.org%2C38147%2C1690029314593.1690029316733, exclude list is [], retry=0 2023-07-22T12:35:16,750 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38553,DS-c76e90c8-39fa-4f11-ba17-53eb00da3429,DISK] 2023-07-22T12:35:16,756 INFO [RS:0;jenkins-hbase3:38147 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/WALs/jenkins-hbase3.apache.org,38147,1690029314593/jenkins-hbase3.apache.org%2C38147%2C1690029314593.1690029316733 2023-07-22T12:35:16,756 DEBUG [RS:0;jenkins-hbase3:38147 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38553,DS-c76e90c8-39fa-4f11-ba17-53eb00da3429,DISK]] 2023-07-22T12:35:16,836 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-07-22T12:35:16,836 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f 2023-07-22T12:35:17,250 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:35:17,253 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-07-22T12:35:17,262 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-07-22T12:35:17,262 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:17,265 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:35:17,265 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-07-22T12:35:17,269 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-07-22T12:35:17,270 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:17,275 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:35:17,275 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-07-22T12:35:17,279 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-07-22T12:35:17,279 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:17,280 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:35:17,282 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/hbase/meta/1588230740 2023-07-22T12:35:17,283 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/hbase/meta/1588230740 2023-07-22T12:35:17,286 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-07-22T12:35:17,289 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-07-22T12:35:17,293 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:35:17,293 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11375335520, jitterRate=0.05941067636013031}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-07-22T12:35:17,296 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-07-22T12:35:17,296 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-07-22T12:35:17,296 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-07-22T12:35:17,296 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-07-22T12:35:17,296 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-07-22T12:35:17,296 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-07-22T12:35:17,298 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-07-22T12:35:17,298 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-07-22T12:35:17,302 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-07-22T12:35:17,302 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-07-22T12:35:17,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-07-22T12:35:17,327 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-07-22T12:35:17,331 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-07-22T12:35:17,484 DEBUG [jenkins-hbase3:46137 {}] assignment.AssignmentManager(2303): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-07-22T12:35:17,492 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:17,499 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,38147,1690029314593, state=OPENING 2023-07-22T12:35:17,508 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-07-22T12:35:17,512 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:35:17,512 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:35:17,513 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-07-22T12:35:17,513 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-07-22T12:35:17,518 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,38147,1690029314593}] 2023-07-22T12:35:17,712 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(721): New admin connection to jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:17,715 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=AdminService, sasl=false 2023-07-22T12:35:17,719 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:33388, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-07-22T12:35:17,732 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-07-22T12:35:17,733 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-07-22T12:35:17,734 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-07-22T12:35:17,775 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C38147%2C1690029314593.meta, suffix=.meta, logDir=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/WALs/jenkins-hbase3.apache.org,38147,1690029314593, archiveDir=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/oldWALs, maxLogs=32 2023-07-22T12:35:17,790 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/WALs/jenkins-hbase3.apache.org,38147,1690029314593/jenkins-hbase3.apache.org%2C38147%2C1690029314593.meta.1690029317777.meta, exclude list is [], retry=0 2023-07-22T12:35:17,795 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38553,DS-c76e90c8-39fa-4f11-ba17-53eb00da3429,DISK] 2023-07-22T12:35:17,800 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/WALs/jenkins-hbase3.apache.org,38147,1690029314593/jenkins-hbase3.apache.org%2C38147%2C1690029314593.meta.1690029317777.meta 2023-07-22T12:35:17,800 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38553,DS-c76e90c8-39fa-4f11-ba17-53eb00da3429,DISK]] 2023-07-22T12:35:17,800 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:35:17,805 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:35:17,806 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-07-22T12:35:17,827 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-07-22T12:35:17,827 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-07-22T12:35:17,833 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-07-22T12:35:17,833 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:35:17,834 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-07-22T12:35:17,834 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-07-22T12:35:17,837 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-07-22T12:35:17,838 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-07-22T12:35:17,838 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:17,839 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:35:17,839 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-07-22T12:35:17,840 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-07-22T12:35:17,841 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:17,841 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:35:17,842 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-07-22T12:35:17,843 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-07-22T12:35:17,843 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:17,844 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:35:17,846 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/hbase/meta/1588230740 2023-07-22T12:35:17,849 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/hbase/meta/1588230740 2023-07-22T12:35:17,852 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-07-22T12:35:17,854 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-07-22T12:35:17,856 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11287718720, jitterRate=0.051250725984573364}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-07-22T12:35:17,857 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-07-22T12:35:17,870 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2552): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1690029317703 2023-07-22T12:35:17,891 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2579): Finished post open deploy task for hbase:meta,,1.1588230740 2023-07-22T12:35:17,892 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-07-22T12:35:17,893 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:17,895 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,38147,1690029314593, state=OPEN 2023-07-22T12:35:17,897 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-07-22T12:35:17,897 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-07-22T12:35:17,898 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-07-22T12:35:17,898 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-07-22T12:35:17,902 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=3, resume processing ppid=2 2023-07-22T12:35:17,902 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,38147,1690029314593 in 379 msec 2023-07-22T12:35:17,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=2, resume processing ppid=1 2023-07-22T12:35:17,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 592 msec 2023-07-22T12:35:17,917 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.6930 sec 2023-07-22T12:35:17,917 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1062): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1690029317917, completionTime=-1 2023-07-22T12:35:17,917 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(830): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-07-22T12:35:17,918 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1629): Joining cluster... 2023-07-22T12:35:18,202 DEBUG [hconnection-0x552edb61-shared-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:35:18,210 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:33400, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:35:18,229 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1641): Number of RegionServers=1 2023-07-22T12:35:18,229 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1690029378229 2023-07-22T12:35:18,229 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1690029438229 2023-07-22T12:35:18,229 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1648): Joined the cluster in 311 msec 2023-07-22T12:35:18,260 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,46137,1690029311532-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-07-22T12:35:18,260 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,46137,1690029311532-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-07-22T12:35:18,260 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,46137,1690029311532-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-07-22T12:35:18,263 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:46137, period=300000, unit=MILLISECONDS is enabled. 2023-07-22T12:35:18,264 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-07-22T12:35:18,272 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-07-22T12:35:18,287 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-07-22T12:35:18,288 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2367): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-07-22T12:35:18,298 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-07-22T12:35:18,303 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:35:18,305 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:18,306 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:35:18,726 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => de72a5b968421854d430063920aa35db, NAME => 'hbase:namespace,,1690029318288.de72a5b968421854d430063920aa35db.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f 2023-07-22T12:35:19,138 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1690029318288.de72a5b968421854d430063920aa35db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:35:19,138 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing de72a5b968421854d430063920aa35db, disabling compactions & flushes 2023-07-22T12:35:19,138 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1690029318288.de72a5b968421854d430063920aa35db. 2023-07-22T12:35:19,138 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1690029318288.de72a5b968421854d430063920aa35db. 2023-07-22T12:35:19,138 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1690029318288.de72a5b968421854d430063920aa35db. after waiting 0 ms 2023-07-22T12:35:19,138 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1690029318288.de72a5b968421854d430063920aa35db. 2023-07-22T12:35:19,138 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1690029318288.de72a5b968421854d430063920aa35db. 2023-07-22T12:35:19,138 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for de72a5b968421854d430063920aa35db: 2023-07-22T12:35:19,142 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:35:19,161 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1690029318288.de72a5b968421854d430063920aa35db.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1690029319144"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029319144"}]},"ts":"1690029319144"} 2023-07-22T12:35:19,198 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:35:19,200 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:35:19,207 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029319200"}]},"ts":"1690029319200"} 2023-07-22T12:35:19,211 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-07-22T12:35:19,217 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=de72a5b968421854d430063920aa35db, ASSIGN}] 2023-07-22T12:35:19,219 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=de72a5b968421854d430063920aa35db, ASSIGN 2023-07-22T12:35:19,221 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=de72a5b968421854d430063920aa35db, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38147,1690029314593; forceNewPlan=false, retain=false 2023-07-22T12:35:19,372 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=de72a5b968421854d430063920aa35db, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:19,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure de72a5b968421854d430063920aa35db, server=jenkins-hbase3.apache.org,38147,1690029314593}] 2023-07-22T12:35:19,536 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1690029318288.de72a5b968421854d430063920aa35db. 2023-07-22T12:35:19,536 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => de72a5b968421854d430063920aa35db, NAME => 'hbase:namespace,,1690029318288.de72a5b968421854d430063920aa35db.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:35:19,537 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:35:19,537 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace de72a5b968421854d430063920aa35db 2023-07-22T12:35:19,537 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1690029318288.de72a5b968421854d430063920aa35db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:35:19,538 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for de72a5b968421854d430063920aa35db 2023-07-22T12:35:19,538 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for de72a5b968421854d430063920aa35db 2023-07-22T12:35:19,540 INFO [StoreOpener-de72a5b968421854d430063920aa35db-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region de72a5b968421854d430063920aa35db 2023-07-22T12:35:19,542 INFO [StoreOpener-de72a5b968421854d430063920aa35db-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de72a5b968421854d430063920aa35db columnFamilyName info 2023-07-22T12:35:19,542 DEBUG [StoreOpener-de72a5b968421854d430063920aa35db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:19,543 INFO [StoreOpener-de72a5b968421854d430063920aa35db-1 {}] regionserver.HStore(324): Store=de72a5b968421854d430063920aa35db/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:35:19,544 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/hbase/namespace/de72a5b968421854d430063920aa35db 2023-07-22T12:35:19,545 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/hbase/namespace/de72a5b968421854d430063920aa35db 2023-07-22T12:35:19,548 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for de72a5b968421854d430063920aa35db 2023-07-22T12:35:19,553 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/hbase/namespace/de72a5b968421854d430063920aa35db/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:35:19,554 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened de72a5b968421854d430063920aa35db; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10531598720, jitterRate=-0.019168436527252197}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:35:19,555 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for de72a5b968421854d430063920aa35db: 2023-07-22T12:35:19,558 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2552): Post open deploy tasks for hbase:namespace,,1690029318288.de72a5b968421854d430063920aa35db., pid=6, masterSystemTime=1690029319529 2023-07-22T12:35:19,561 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2579): Finished post open deploy task for hbase:namespace,,1690029318288.de72a5b968421854d430063920aa35db. 2023-07-22T12:35:19,561 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1690029318288.de72a5b968421854d430063920aa35db. 2023-07-22T12:35:19,563 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=de72a5b968421854d430063920aa35db, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:19,572 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=6, resume processing ppid=5 2023-07-22T12:35:19,572 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure de72a5b968421854d430063920aa35db, server=jenkins-hbase3.apache.org,38147,1690029314593 in 193 msec 2023-07-22T12:35:19,576 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=5, resume processing ppid=4 2023-07-22T12:35:19,576 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=de72a5b968421854d430063920aa35db, ASSIGN in 355 msec 2023-07-22T12:35:19,577 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:35:19,578 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029319577"}]},"ts":"1690029319577"} 2023-07-22T12:35:19,581 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-07-22T12:35:19,585 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:35:19,589 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 1.2950 sec 2023-07-22T12:35:19,603 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Set watcher on znode that does not yet exist, /1/namespace 2023-07-22T12:35:19,606 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/namespace 2023-07-22T12:35:19,606 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:35:19,606 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:35:19,653 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-07-22T12:35:19,668 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:35:19,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 29 msec 2023-07-22T12:35:19,677 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-07-22T12:35:19,688 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:35:19,693 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 15 msec 2023-07-22T12:35:19,703 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/default 2023-07-22T12:35:19,706 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/hbase 2023-07-22T12:35:19,706 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1192): Master has completed initialization 4.982sec 2023-07-22T12:35:19,708 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-07-22T12:35:19,710 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-07-22T12:35:19,711 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-07-22T12:35:19,712 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-07-22T12:35:19,712 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-07-22T12:35:19,714 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,46137,1690029311532-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-07-22T12:35:19,714 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,46137,1690029311532-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-07-22T12:35:19,720 DEBUG [Listener at localhost/39493 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x443737b3 to 127.0.0.1:54609 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:35:19,724 DEBUG [Listener at localhost/39493 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29070799, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:35:19,728 DEBUG [Listener at localhost/39493 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-07-22T12:35:19,728 DEBUG [Listener at localhost/39493 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-07-22T12:35:19,735 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1295): Balancer post startup initialization complete, took 0 seconds 2023-07-22T12:35:19,737 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-07-22T12:35:19,745 DEBUG [hconnection-0xe55d0ff-shared-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:35:19,756 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:33412, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:35:19,766 INFO [Listener at localhost/39493 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,46137,1690029311532 2023-07-22T12:35:19,778 INFO [Listener at localhost/39493 {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/hadoop.log.dir so I do NOT create it in target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a 2023-07-22T12:35:19,779 INFO [Listener at localhost/39493 {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/hadoop.tmp.dir so I do NOT create it in target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a 2023-07-22T12:35:19,779 DEBUG [Listener at localhost/39493 {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a 2023-07-22T12:35:19,779 INFO [Listener at localhost/39493 {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/cluster_3623d274-aa27-74cf-d526-f07dcf62976b, deleteOnExit=true 2023-07-22T12:35:19,779 INFO [Listener at localhost/39493 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/test.cache.data in system properties and HBase conf 2023-07-22T12:35:19,779 INFO [Listener at localhost/39493 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/hadoop.tmp.dir in system properties and HBase conf 2023-07-22T12:35:19,779 INFO [Listener at localhost/39493 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/hadoop.log.dir in system properties and HBase conf 2023-07-22T12:35:19,779 INFO [Listener at localhost/39493 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/mapreduce.cluster.local.dir in system properties and HBase conf 2023-07-22T12:35:19,779 INFO [Listener at localhost/39493 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-07-22T12:35:19,779 INFO [Listener at localhost/39493 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-07-22T12:35:19,780 DEBUG [Listener at localhost/39493 {}] fs.HFileSystem(308): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-07-22T12:35:19,780 INFO [Listener at localhost/39493 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-07-22T12:35:19,780 INFO [Listener at localhost/39493 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-07-22T12:35:19,780 INFO [Listener at localhost/39493 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-07-22T12:35:19,780 INFO [Listener at localhost/39493 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-07-22T12:35:19,780 INFO [Listener at localhost/39493 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-07-22T12:35:19,780 INFO [Listener at localhost/39493 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-07-22T12:35:19,780 INFO [Listener at localhost/39493 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-07-22T12:35:19,781 INFO [Listener at localhost/39493 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/dfs.journalnode.edits.dir in system properties and HBase conf 2023-07-22T12:35:19,781 INFO [Listener at localhost/39493 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-07-22T12:35:19,781 INFO [Listener at localhost/39493 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/nfs.dump.dir in system properties and HBase conf 2023-07-22T12:35:19,781 INFO [Listener at localhost/39493 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/java.io.tmpdir in system properties and HBase conf 2023-07-22T12:35:19,781 INFO [Listener at localhost/39493 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/dfs.journalnode.edits.dir in system properties and HBase conf 2023-07-22T12:35:19,781 INFO [Listener at localhost/39493 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-07-22T12:35:19,781 INFO [Listener at localhost/39493 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-07-22T12:35:19,796 WARN [Listener at localhost/39493 {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-07-22T12:35:19,796 WARN [Listener at localhost/39493 {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-07-22T12:35:19,840 WARN [Listener at localhost/39493 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:35:19,843 INFO [Listener at localhost/39493 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:35:19,849 INFO [Listener at localhost/39493 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/java.io.tmpdir/Jetty_localhost_35311_hdfs____8tbw63/webapp 2023-07-22T12:35:19,956 INFO [Listener at localhost/39493 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:35311 2023-07-22T12:35:19,970 WARN [Listener at localhost/39493 {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-07-22T12:35:19,970 WARN [Listener at localhost/39493 {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-07-22T12:35:20,012 WARN [Listener at localhost/38161 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-07-22T12:35:20,027 WARN [Listener at localhost/38161 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-07-22T12:35:20,031 WARN [Listener at localhost/38161 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:35:20,033 INFO [Listener at localhost/38161 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:35:20,039 INFO [Listener at localhost/38161 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/java.io.tmpdir/Jetty_localhost_34249_datanode____rjmzh/webapp 2023-07-22T12:35:20,151 INFO [Listener at localhost/38161 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:34249 2023-07-22T12:35:20,159 WARN [Listener at localhost/43257 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-07-22T12:35:20,225 WARN [Listener at localhost/43257 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-07-22T12:35:20,229 WARN [Listener at localhost/43257 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:35:20,231 INFO [Listener at localhost/43257 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:35:20,234 INFO [Listener at localhost/43257 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/java.io.tmpdir/Jetty_localhost_43489_datanode____.n0r54z/webapp 2023-07-22T12:35:20,285 WARN [Thread-202 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-07-22T12:35:20,326 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xe6b8475b7107a90: Processing first storage report for DS-e8126e2c-aa25-4919-967e-8d7c8d982603 from datanode 6e1b725d-9d4d-4c66-9493-606eafe4ca96 2023-07-22T12:35:20,326 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xe6b8475b7107a90: from storage DS-e8126e2c-aa25-4919-967e-8d7c8d982603 node DatanodeRegistration(127.0.0.1:37139, datanodeUuid=6e1b725d-9d4d-4c66-9493-606eafe4ca96, infoPort=41217, infoSecurePort=0, ipcPort=43257, storageInfo=lv=-57;cid=testClusterID;nsid=1468785099;c=1690029319799), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2023-07-22T12:35:20,326 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xe6b8475b7107a90: Processing first storage report for DS-50c03aaf-49d6-4605-8ea9-30289fd316e3 from datanode 6e1b725d-9d4d-4c66-9493-606eafe4ca96 2023-07-22T12:35:20,327 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xe6b8475b7107a90: from storage DS-50c03aaf-49d6-4605-8ea9-30289fd316e3 node DatanodeRegistration(127.0.0.1:37139, datanodeUuid=6e1b725d-9d4d-4c66-9493-606eafe4ca96, infoPort=41217, infoSecurePort=0, ipcPort=43257, storageInfo=lv=-57;cid=testClusterID;nsid=1468785099;c=1690029319799), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-07-22T12:35:20,356 INFO [Listener at localhost/43257 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:43489 2023-07-22T12:35:20,365 WARN [Listener at localhost/44409 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-07-22T12:35:20,396 WARN [Listener at localhost/44409 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-07-22T12:35:20,400 WARN [Listener at localhost/44409 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:35:20,402 INFO [Listener at localhost/44409 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:35:20,411 INFO [Listener at localhost/44409 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/java.io.tmpdir/Jetty_localhost_40105_datanode____dt5srl/webapp 2023-07-22T12:35:20,456 WARN [Thread-237 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-07-22T12:35:20,495 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x36846f9498c013ef: Processing first storage report for DS-8e397e6d-70a5-40e9-abf4-d0e78c803d2d from datanode 04e3e809-64d1-429d-8b2c-007ae22876f6 2023-07-22T12:35:20,495 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x36846f9498c013ef: from storage DS-8e397e6d-70a5-40e9-abf4-d0e78c803d2d node DatanodeRegistration(127.0.0.1:34799, datanodeUuid=04e3e809-64d1-429d-8b2c-007ae22876f6, infoPort=37539, infoSecurePort=0, ipcPort=44409, storageInfo=lv=-57;cid=testClusterID;nsid=1468785099;c=1690029319799), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2023-07-22T12:35:20,495 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x36846f9498c013ef: Processing first storage report for DS-a65a81db-acd3-44c0-9c3b-84a4234d3ef8 from datanode 04e3e809-64d1-429d-8b2c-007ae22876f6 2023-07-22T12:35:20,496 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x36846f9498c013ef: from storage DS-a65a81db-acd3-44c0-9c3b-84a4234d3ef8 node DatanodeRegistration(127.0.0.1:34799, datanodeUuid=04e3e809-64d1-429d-8b2c-007ae22876f6, infoPort=37539, infoSecurePort=0, ipcPort=44409, storageInfo=lv=-57;cid=testClusterID;nsid=1468785099;c=1690029319799), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-07-22T12:35:20,534 INFO [Listener at localhost/44409 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:40105 2023-07-22T12:35:20,552 WARN [Listener at localhost/42321 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-07-22T12:35:20,669 WARN [Thread-271 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-07-22T12:35:20,701 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x946a6a0925570ee3: Processing first storage report for DS-593ebd56-44b5-4a5f-a6ed-3e53df66d84b from datanode a99da7a0-b8a1-481d-bfd6-37121c4b89bf 2023-07-22T12:35:20,701 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x946a6a0925570ee3: from storage DS-593ebd56-44b5-4a5f-a6ed-3e53df66d84b node DatanodeRegistration(127.0.0.1:46725, datanodeUuid=a99da7a0-b8a1-481d-bfd6-37121c4b89bf, infoPort=44009, infoSecurePort=0, ipcPort=42321, storageInfo=lv=-57;cid=testClusterID;nsid=1468785099;c=1690029319799), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2023-07-22T12:35:20,701 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x946a6a0925570ee3: Processing first storage report for DS-374f4e3c-2c43-49f1-ad97-0aad88c79da1 from datanode a99da7a0-b8a1-481d-bfd6-37121c4b89bf 2023-07-22T12:35:20,701 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x946a6a0925570ee3: from storage DS-374f4e3c-2c43-49f1-ad97-0aad88c79da1 node DatanodeRegistration(127.0.0.1:46725, datanodeUuid=a99da7a0-b8a1-481d-bfd6-37121c4b89bf, infoPort=44009, infoSecurePort=0, ipcPort=42321, storageInfo=lv=-57;cid=testClusterID;nsid=1468785099;c=1690029319799), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-07-22T12:35:20,779 DEBUG [Listener at localhost/42321 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a 2023-07-22T12:35:20,779 INFO [Listener at localhost/42321 {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-07-22T12:35:20,779 INFO [Listener at localhost/42321 {}] hbase.HBaseTestingUtility(1143): NOT STARTING DFS 2023-07-22T12:35:20,779 INFO [Listener at localhost/42321 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:35:20,780 INFO [Listener at localhost/42321 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:35:21,203 INFO [Listener at localhost/42321 {}] util.FSUtils(461): Created version file at hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a with version=8 2023-07-22T12:35:21,203 INFO [Listener at localhost/42321 {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/hbase-staging 2023-07-22T12:35:21,204 DEBUG [Listener at localhost/42321 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-07-22T12:35:21,204 DEBUG [Listener at localhost/42321 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-07-22T12:35:21,204 DEBUG [Listener at localhost/42321 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-07-22T12:35:21,204 DEBUG [Listener at localhost/42321 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-07-22T12:35:21,205 INFO [Listener at localhost/42321 {}] client.ConnectionUtils(127): master/jenkins-hbase3:0 server-side Connection retries=45 2023-07-22T12:35:21,205 INFO [Listener at localhost/42321 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-07-22T12:35:21,205 INFO [Listener at localhost/42321 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-07-22T12:35:21,205 INFO [Listener at localhost/42321 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-07-22T12:35:21,206 INFO [Listener at localhost/42321 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-07-22T12:35:21,206 INFO [Listener at localhost/42321 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-07-22T12:35:21,206 INFO [Listener at localhost/42321 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-07-22T12:35:21,206 INFO [Listener at localhost/42321 {}] ipc.NettyRpcServer(177): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-07-22T12:35:21,207 INFO [Listener at localhost/42321 {}] ipc.NettyRpcServer(144): Bind to /172.31.12.81:46439 2023-07-22T12:35:21,207 INFO [Listener at localhost/42321 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:35:21,208 INFO [Listener at localhost/42321 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:35:21,210 INFO [Listener at localhost/42321 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:46439 connecting to ZooKeeper ensemble=127.0.0.1:54609 2023-07-22T12:35:21,218 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): master:464390x0, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-07-22T12:35:21,219 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:46439-0x100006222d70004 connected 2023-07-22T12:35:21,230 DEBUG [Listener at localhost/42321 {}] zookeeper.ZKUtil(113): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Set watcher on znode that does not yet exist, /2/master 2023-07-22T12:35:21,231 DEBUG [Listener at localhost/42321 {}] zookeeper.ZKUtil(113): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Set watcher on znode that does not yet exist, /2/running 2023-07-22T12:35:21,231 DEBUG [Listener at localhost/42321 {}] zookeeper.ZKUtil(113): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Set watcher on znode that does not yet exist, /2/acl 2023-07-22T12:35:21,232 DEBUG [Listener at localhost/42321 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46439 2023-07-22T12:35:21,232 DEBUG [Listener at localhost/42321 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46439 2023-07-22T12:35:21,232 DEBUG [Listener at localhost/42321 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46439 2023-07-22T12:35:21,232 DEBUG [Listener at localhost/42321 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46439 2023-07-22T12:35:21,233 DEBUG [Listener at localhost/42321 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46439 2023-07-22T12:35:21,236 INFO [Listener at localhost/42321 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-07-22T12:35:21,236 INFO [Listener at localhost/42321 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-07-22T12:35:21,236 INFO [Listener at localhost/42321 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-07-22T12:35:21,236 INFO [Listener at localhost/42321 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-07-22T12:35:21,236 INFO [Listener at localhost/42321 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-07-22T12:35:21,236 INFO [Listener at localhost/42321 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-07-22T12:35:21,236 INFO [Listener at localhost/42321 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-07-22T12:35:21,238 INFO [Listener at localhost/42321 {}] http.HttpServer(1219): Jetty bound to port 41301 2023-07-22T12:35:21,238 INFO [Listener at localhost/42321 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-07-22T12:35:21,242 INFO [Listener at localhost/42321 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:35:21,242 INFO [Listener at localhost/42321 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@7668c2d0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/hadoop.log.dir/,AVAILABLE} 2023-07-22T12:35:21,242 INFO [Listener at localhost/42321 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:35:21,242 INFO [Listener at localhost/42321 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@9a76e17{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-07-22T12:35:21,252 INFO [Listener at localhost/42321 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-07-22T12:35:21,254 INFO [Listener at localhost/42321 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-07-22T12:35:21,254 INFO [Listener at localhost/42321 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-07-22T12:35:21,254 INFO [Listener at localhost/42321 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2023-07-22T12:35:21,255 INFO [Listener at localhost/42321 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:35:21,342 INFO [Listener at localhost/42321 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@42db99be{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-07-22T12:35:21,344 INFO [Listener at localhost/42321 {}] server.AbstractConnector(333): Started ServerConnector@7977b13c{HTTP/1.1, (http/1.1)}{0.0.0.0:41301} 2023-07-22T12:35:21,344 INFO [Listener at localhost/42321 {}] server.Server(415): Started @15081ms 2023-07-22T12:35:21,344 INFO [Listener at localhost/42321 {}] master.HMaster(484): hbase.rootdir=hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a, hbase.cluster.distributed=false 2023-07-22T12:35:21,363 INFO [Listener at localhost/42321 {}] client.ConnectionUtils(127): regionserver/jenkins-hbase3:0 server-side Connection retries=45 2023-07-22T12:35:21,363 INFO [Listener at localhost/42321 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-07-22T12:35:21,363 INFO [Listener at localhost/42321 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-07-22T12:35:21,363 INFO [Listener at localhost/42321 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-07-22T12:35:21,364 INFO [Listener at localhost/42321 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-07-22T12:35:21,364 INFO [Listener at localhost/42321 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-07-22T12:35:21,364 INFO [Listener at localhost/42321 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-07-22T12:35:21,364 INFO [Listener at localhost/42321 {}] ipc.NettyRpcServer(177): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-07-22T12:35:21,366 INFO [Listener at localhost/42321 {}] ipc.NettyRpcServer(144): Bind to /172.31.12.81:39575 2023-07-22T12:35:21,366 INFO [Listener at localhost/42321 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-07-22T12:35:21,371 DEBUG [Listener at localhost/42321 {}] mob.MobFileCache(120): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-07-22T12:35:21,372 INFO [Listener at localhost/42321 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:35:21,374 INFO [Listener at localhost/42321 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:35:21,375 INFO [Listener at localhost/42321 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:39575 connecting to ZooKeeper ensemble=127.0.0.1:54609 2023-07-22T12:35:21,379 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): regionserver:395750x0, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-07-22T12:35:21,381 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:39575-0x100006222d70005 connected 2023-07-22T12:35:21,381 DEBUG [Listener at localhost/42321 {}] zookeeper.ZKUtil(113): regionserver:39575-0x100006222d70005, quorum=127.0.0.1:54609, baseZNode=/2 Set watcher on znode that does not yet exist, /2/master 2023-07-22T12:35:21,382 DEBUG [Listener at localhost/42321 {}] zookeeper.ZKUtil(113): regionserver:39575-0x100006222d70005, quorum=127.0.0.1:54609, baseZNode=/2 Set watcher on znode that does not yet exist, /2/running 2023-07-22T12:35:21,382 DEBUG [Listener at localhost/42321 {}] zookeeper.ZKUtil(113): regionserver:39575-0x100006222d70005, quorum=127.0.0.1:54609, baseZNode=/2 Set watcher on znode that does not yet exist, /2/acl 2023-07-22T12:35:21,389 DEBUG [Listener at localhost/42321 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39575 2023-07-22T12:35:21,389 DEBUG [Listener at localhost/42321 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39575 2023-07-22T12:35:21,389 DEBUG [Listener at localhost/42321 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39575 2023-07-22T12:35:21,391 DEBUG [Listener at localhost/42321 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39575 2023-07-22T12:35:21,391 DEBUG [Listener at localhost/42321 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39575 2023-07-22T12:35:21,396 INFO [Listener at localhost/42321 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-07-22T12:35:21,396 INFO [Listener at localhost/42321 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-07-22T12:35:21,397 INFO [Listener at localhost/42321 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-07-22T12:35:21,397 INFO [Listener at localhost/42321 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-07-22T12:35:21,397 INFO [Listener at localhost/42321 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-07-22T12:35:21,398 INFO [Listener at localhost/42321 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-07-22T12:35:21,398 INFO [Listener at localhost/42321 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-07-22T12:35:21,399 INFO [Listener at localhost/42321 {}] http.HttpServer(1219): Jetty bound to port 36663 2023-07-22T12:35:21,399 INFO [Listener at localhost/42321 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-07-22T12:35:21,402 INFO [Listener at localhost/42321 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:35:21,403 INFO [Listener at localhost/42321 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@44bed0d6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/hadoop.log.dir/,AVAILABLE} 2023-07-22T12:35:21,403 INFO [Listener at localhost/42321 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:35:21,403 INFO [Listener at localhost/42321 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@38041cf0{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-07-22T12:35:21,411 INFO [Listener at localhost/42321 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-07-22T12:35:21,411 INFO [Listener at localhost/42321 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-07-22T12:35:21,411 INFO [Listener at localhost/42321 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-07-22T12:35:21,412 INFO [Listener at localhost/42321 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-07-22T12:35:21,412 INFO [Listener at localhost/42321 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:35:21,413 INFO [Listener at localhost/42321 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@13f002cd{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-07-22T12:35:21,414 INFO [Listener at localhost/42321 {}] server.AbstractConnector(333): Started ServerConnector@2f51b5c0{HTTP/1.1, (http/1.1)}{0.0.0.0:36663} 2023-07-22T12:35:21,414 INFO [Listener at localhost/42321 {}] server.Server(415): Started @15151ms 2023-07-22T12:35:21,416 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-07-22T12:35:21,420 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@10c22c07{HTTP/1.1, (http/1.1)}{0.0.0.0:39415} 2023-07-22T12:35:21,421 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(415): Started @15157ms 2023-07-22T12:35:21,421 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2387): Adding backup master ZNode /2/backup-masters/jenkins-hbase3.apache.org,46439,1690029321204 2023-07-22T12:35:21,424 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39575-0x100006222d70005, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/backup-masters 2023-07-22T12:35:21,424 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/backup-masters 2023-07-22T12:35:21,424 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Set watcher on existing znode=/2/backup-masters/jenkins-hbase3.apache.org,46439,1690029321204 2023-07-22T12:35:21,427 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/2/master 2023-07-22T12:35:21,427 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-07-22T12:35:21,427 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39575-0x100006222d70005, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/2/master 2023-07-22T12:35:21,427 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39575-0x100006222d70005, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-07-22T12:35:21,428 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Set watcher on existing znode=/2/master 2023-07-22T12:35:21,430 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /2/backup-masters/jenkins-hbase3.apache.org,46439,1690029321204 from backup master directory 2023-07-22T12:35:21,431 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39575-0x100006222d70005, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/backup-masters 2023-07-22T12:35:21,432 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-07-22T12:35:21,432 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/2/backup-masters/jenkins-hbase3.apache.org,46439,1690029321204 2023-07-22T12:35:21,432 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Set watcher on existing znode=/2/master 2023-07-22T12:35:21,432 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,46439,1690029321204 2023-07-22T12:35:21,432 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/backup-masters 2023-07-22T12:35:21,432 DEBUG [M:0;jenkins-hbase3:46439 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:46439 2023-07-22T12:35:21,692 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-07-22T12:35:21,693 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-07-22T12:35:21,699 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-07-22T12:35:21,700 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-07-22T12:35:21,701 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:35:21,701 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2023-07-22T12:35:21,701 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-07-22T12:35:21,701 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-07-22T12:35:21,701 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-07-22T12:35:21,701 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-07-22T12:35:21,855 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(610): Created cluster ID file at hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/hbase.id with ID: b2d03b55-9660-4abb-8cf0-55cf61c3500d 2023-07-22T12:35:21,873 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:35:21,877 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39575-0x100006222d70005, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-07-22T12:35:21,878 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-07-22T12:35:21,898 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:35:21,898 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-07-22T12:35:21,899 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-07-22T12:35:22,315 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/MasterData/data/master/store 2023-07-22T12:35:22,534 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-07-22T12:35:22,535 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-07-22T12:35:22,729 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:35:22,729 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-07-22T12:35:22,729 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:35:22,729 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:35:22,729 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-07-22T12:35:22,729 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:35:22,729 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:35:22,729 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-07-22T12:35:22,731 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/MasterData/data/master/store/.initializing 2023-07-22T12:35:22,731 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:38161/tmp/wal/MasterData/WALs/jenkins-hbase3.apache.org,46439,1690029321204 2023-07-22T12:35:22,733 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-07-22T12:35:22,736 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C46439%2C1690029321204, suffix=, logDir=hdfs://localhost:38161/tmp/wal/MasterData/WALs/jenkins-hbase3.apache.org,46439,1690029321204, archiveDir=hdfs://localhost:38161/tmp/wal/MasterData/oldWALs, maxLogs=10 2023-07-22T12:35:22,747 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /tmp/wal/MasterData/WALs/jenkins-hbase3.apache.org,46439,1690029321204/jenkins-hbase3.apache.org%2C46439%2C1690029321204.1690029322736, exclude list is [], retry=0 2023-07-22T12:35:22,751 DEBUG [RS-EventLoopGroup-6-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46725,DS-593ebd56-44b5-4a5f-a6ed-3e53df66d84b,DISK] 2023-07-22T12:35:22,755 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /tmp/wal/MasterData/WALs/jenkins-hbase3.apache.org,46439,1690029321204/jenkins-hbase3.apache.org%2C46439%2C1690029321204.1690029322736 2023-07-22T12:35:22,756 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:46725,DS-593ebd56-44b5-4a5f-a6ed-3e53df66d84b,DISK]] 2023-07-22T12:35:22,756 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:35:22,756 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:35:22,756 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:35:22,756 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:35:22,761 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:35:22,763 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-07-22T12:35:22,764 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:22,765 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:35:22,765 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:35:22,766 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-07-22T12:35:22,766 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:22,767 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:35:22,767 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:35:22,769 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-07-22T12:35:22,769 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:22,770 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:35:22,770 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:35:22,772 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-07-22T12:35:22,772 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:22,772 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:35:22,773 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:35:22,774 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:35:22,775 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38161/tmp/wal/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:35:22,777 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-07-22T12:35:22,779 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:35:22,782 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38161/tmp/wal/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:35:22,783 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11858028960, jitterRate=0.10436500608921051}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-07-22T12:35:22,783 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-07-22T12:35:22,783 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-07-22T12:35:22,787 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6478321d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:35:22,788 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(859): No meta location available on zookeeper, skip migrating... 2023-07-22T12:35:22,789 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-07-22T12:35:22,789 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(564): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-07-22T12:35:22,789 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-07-22T12:35:22,790 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(584): Recovered RegionProcedureStore lease in 0 msec 2023-07-22T12:35:22,790 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(598): Loaded RegionProcedureStore in 0 msec 2023-07-22T12:35:22,790 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-07-22T12:35:22,792 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-07-22T12:35:22,794 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Unable to get data of znode /2/balancer because node does not exist (not necessarily an error) 2023-07-22T12:35:22,795 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /2/balancer already deleted, retry=false 2023-07-22T12:35:22,795 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(150): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-07-22T12:35:22,796 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Unable to get data of znode /2/normalizer because node does not exist (not necessarily an error) 2023-07-22T12:35:22,797 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /2/normalizer already deleted, retry=false 2023-07-22T12:35:22,797 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-07-22T12:35:22,798 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Unable to get data of znode /2/switch/split because node does not exist (not necessarily an error) 2023-07-22T12:35:22,800 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /2/switch/split already deleted, retry=false 2023-07-22T12:35:22,801 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Unable to get data of znode /2/switch/merge because node does not exist (not necessarily an error) 2023-07-22T12:35:22,802 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /2/switch/merge already deleted, retry=false 2023-07-22T12:35:22,804 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Unable to get data of znode /2/snapshot-cleanup because node does not exist (not necessarily an error) 2023-07-22T12:35:22,805 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /2/snapshot-cleanup already deleted, retry=false 2023-07-22T12:35:22,806 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/2/running 2023-07-22T12:35:22,806 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39575-0x100006222d70005, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/2/running 2023-07-22T12:35:22,806 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-07-22T12:35:22,806 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39575-0x100006222d70005, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-07-22T12:35:22,807 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(803): Active/primary master=jenkins-hbase3.apache.org,46439,1690029321204, sessionid=0x100006222d70004, setting cluster-up flag (Was=false) 2023-07-22T12:35:22,807 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-07-22T12:35:22,811 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-07-22T12:35:22,811 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39575-0x100006222d70005, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-07-22T12:35:22,818 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /2/flush-table-proc/acquired, /2/flush-table-proc/reached, /2/flush-table-proc/abort 2023-07-22T12:35:22,820 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,46439,1690029321204 2023-07-22T12:35:22,830 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-07-22T12:35:22,830 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39575-0x100006222d70005, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-07-22T12:35:22,835 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /2/rolllog-proc/acquired, /2/rolllog-proc/reached, /2/rolllog-proc/abort 2023-07-22T12:35:22,836 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,46439,1690029321204 2023-07-22T12:35:22,840 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39575-0x100006222d70005, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-07-22T12:35:22,840 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-07-22T12:35:22,857 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /2/online-snapshot/acquired, /2/online-snapshot/reached, /2/online-snapshot/abort 2023-07-22T12:35:22,860 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,46439,1690029321204 2023-07-22T12:35:22,866 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-07-22T12:35:22,867 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-07-22T12:35:22,867 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-07-22T12:35:22,868 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,46439,1690029321204 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-07-22T12:35:22,868 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-07-22T12:35:22,868 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-07-22T12:35:22,868 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-07-22T12:35:22,868 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-07-22T12:35:22,869 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-07-22T12:35:22,869 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:35:22,869 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-07-22T12:35:22,869 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:35:22,878 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1690029352877 2023-07-22T12:35:22,880 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-07-22T12:35:22,882 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-07-22T12:35:22,882 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-07-22T12:35:22,882 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-07-22T12:35:22,882 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-07-22T12:35:22,882 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-07-22T12:35:22,882 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-07-22T12:35:22,882 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-07-22T12:35:22,888 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-07-22T12:35:22,888 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-07-22T12:35:22,888 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-07-22T12:35:22,888 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-07-22T12:35:22,888 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-07-22T12:35:22,888 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5f7bc8f4 to 127.0.0.1:54609 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:35:22,891 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:22,891 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-07-22T12:35:22,893 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e36f247, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:35:22,893 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-07-22T12:35:22,898 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-07-22T12:35:22,898 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-07-22T12:35:22,898 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1690029322898,5,FailOnTimeoutGroup] 2023-07-22T12:35:22,899 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1690029322898,5,FailOnTimeoutGroup] 2023-07-22T12:35:22,899 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-07-22T12:35:22,899 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1634): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-07-22T12:35:22,899 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-07-22T12:35:22,899 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-07-22T12:35:22,925 DEBUG [RS:0;jenkins-hbase3:39575 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:39575 2023-07-22T12:35:22,926 INFO [RS:0;jenkins-hbase3:39575 {}] regionserver.HRegionServer(997): ClusterId : b2d03b55-9660-4abb-8cf0-55cf61c3500d 2023-07-22T12:35:22,926 INFO [RS:0;jenkins-hbase3:39575 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-07-22T12:35:22,927 DEBUG [RS:0;jenkins-hbase3:39575 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-07-22T12:35:22,929 DEBUG [RS:0;jenkins-hbase3:39575 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-07-22T12:35:22,929 DEBUG [RS:0;jenkins-hbase3:39575 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-07-22T12:35:22,932 DEBUG [RS:0;jenkins-hbase3:39575 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-07-22T12:35:22,932 DEBUG [RS:0;jenkins-hbase3:39575 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-07-22T12:35:22,934 DEBUG [RS:0;jenkins-hbase3:39575 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-07-22T12:35:22,935 DEBUG [RS:0;jenkins-hbase3:39575 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@961cb46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:35:22,936 DEBUG [RS:0;jenkins-hbase3:39575 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c8db9c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-07-22T12:35:22,936 INFO [RS:0;jenkins-hbase3:39575 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-07-22T12:35:22,936 INFO [RS:0;jenkins-hbase3:39575 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-07-22T12:35:22,936 DEBUG [RS:0;jenkins-hbase3:39575 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-07-22T12:35:22,937 INFO [RS:0;jenkins-hbase3:39575 {}] regionserver.HRegionServer(3023): reportForDuty to master=jenkins-hbase3.apache.org,46439,1690029321204 with isa=jenkins-hbase3.apache.org/172.31.12.81:39575, startcode=1690029321362 2023-07-22T12:35:22,937 DEBUG [RS:0;jenkins-hbase3:39575 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-07-22T12:35:22,940 INFO [RS-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:39041, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2023-07-22T12:35:22,942 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46439 {}] master.ServerManager(403): Registering regionserver=jenkins-hbase3.apache.org,39575,1690029321362 2023-07-22T12:35:22,945 DEBUG [RS:0;jenkins-hbase3:39575 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a 2023-07-22T12:35:22,945 DEBUG [RS:0;jenkins-hbase3:39575 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:38161 2023-07-22T12:35:22,945 DEBUG [RS:0;jenkins-hbase3:39575 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=41301 2023-07-22T12:35:22,947 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/rs 2023-07-22T12:35:22,947 DEBUG [RS:0;jenkins-hbase3:39575 {}] zookeeper.ZKUtil(111): regionserver:39575-0x100006222d70005, quorum=127.0.0.1:54609, baseZNode=/2 Set watcher on existing znode=/2/rs/jenkins-hbase3.apache.org,39575,1690029321362 2023-07-22T12:35:22,947 WARN [RS:0;jenkins-hbase3:39575 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-07-22T12:35:22,947 INFO [RS:0;jenkins-hbase3:39575 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-07-22T12:35:22,947 DEBUG [RS:0;jenkins-hbase3:39575 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:38161/tmp/wal/WALs/jenkins-hbase3.apache.org,39575,1690029321362 2023-07-22T12:35:22,950 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,39575,1690029321362] 2023-07-22T12:35:22,956 DEBUG [RS:0;jenkins-hbase3:39575 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-07-22T12:35:22,956 INFO [RS:0;jenkins-hbase3:39575 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-07-22T12:35:22,958 INFO [RS:0;jenkins-hbase3:39575 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-07-22T12:35:22,962 INFO [RS:0;jenkins-hbase3:39575 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-07-22T12:35:22,962 INFO [RS:0;jenkins-hbase3:39575 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-07-22T12:35:22,965 INFO [RS:0;jenkins-hbase3:39575 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-07-22T12:35:22,967 INFO [RS:0;jenkins-hbase3:39575 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-07-22T12:35:22,967 DEBUG [RS:0;jenkins-hbase3:39575 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:35:22,967 DEBUG [RS:0;jenkins-hbase3:39575 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:35:22,967 DEBUG [RS:0;jenkins-hbase3:39575 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:35:22,968 DEBUG [RS:0;jenkins-hbase3:39575 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:35:22,968 DEBUG [RS:0;jenkins-hbase3:39575 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:35:22,968 DEBUG [RS:0;jenkins-hbase3:39575 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-07-22T12:35:22,968 DEBUG [RS:0;jenkins-hbase3:39575 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:35:22,969 DEBUG [RS:0;jenkins-hbase3:39575 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:35:22,969 DEBUG [RS:0;jenkins-hbase3:39575 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:35:22,969 DEBUG [RS:0;jenkins-hbase3:39575 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:35:22,969 DEBUG [RS:0;jenkins-hbase3:39575 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:35:22,969 DEBUG [RS:0;jenkins-hbase3:39575 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-07-22T12:35:22,978 INFO [RS:0;jenkins-hbase3:39575 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-07-22T12:35:22,978 INFO [RS:0;jenkins-hbase3:39575 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-07-22T12:35:22,978 INFO [RS:0;jenkins-hbase3:39575 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-07-22T12:35:22,978 INFO [RS:0;jenkins-hbase3:39575 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-07-22T12:35:22,978 INFO [RS:0;jenkins-hbase3:39575 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,39575,1690029321362-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-07-22T12:35:22,995 INFO [RS:0;jenkins-hbase3:39575 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-07-22T12:35:22,995 INFO [RS:0;jenkins-hbase3:39575 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,39575,1690029321362-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-07-22T12:35:23,012 INFO [RS:0;jenkins-hbase3:39575 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,39575,1690029321362 started 2023-07-22T12:35:23,012 INFO [RS:0;jenkins-hbase3:39575 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,39575,1690029321362, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:39575, sessionid=0x100006222d70005 2023-07-22T12:35:23,012 DEBUG [RS:0;jenkins-hbase3:39575 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-07-22T12:35:23,013 DEBUG [RS:0;jenkins-hbase3:39575 {}] flush.RegionServerFlushTableProcedureManager(106): Start region server flush procedure manager jenkins-hbase3.apache.org,39575,1690029321362 2023-07-22T12:35:23,013 DEBUG [RS:0;jenkins-hbase3:39575 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,39575,1690029321362' 2023-07-22T12:35:23,013 DEBUG [RS:0;jenkins-hbase3:39575 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/2/flush-table-proc/abort' 2023-07-22T12:35:23,013 DEBUG [RS:0;jenkins-hbase3:39575 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/2/flush-table-proc/acquired' 2023-07-22T12:35:23,014 DEBUG [RS:0;jenkins-hbase3:39575 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-07-22T12:35:23,014 DEBUG [RS:0;jenkins-hbase3:39575 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-07-22T12:35:23,014 DEBUG [RS:0;jenkins-hbase3:39575 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,39575,1690029321362' 2023-07-22T12:35:23,014 DEBUG [RS:0;jenkins-hbase3:39575 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/2/rolllog-proc/abort' 2023-07-22T12:35:23,014 DEBUG [RS:0;jenkins-hbase3:39575 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/2/rolllog-proc/acquired' 2023-07-22T12:35:23,015 INFO [RS:0;jenkins-hbase3:39575 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-07-22T12:35:23,015 DEBUG [RS:0;jenkins-hbase3:39575 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-07-22T12:35:23,015 DEBUG [RS:0;jenkins-hbase3:39575 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-07-22T12:35:23,015 DEBUG [RS:0;jenkins-hbase3:39575 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,39575,1690029321362 2023-07-22T12:35:23,015 DEBUG [RS:0;jenkins-hbase3:39575 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,39575,1690029321362' 2023-07-22T12:35:23,015 DEBUG [RS:0;jenkins-hbase3:39575 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/2/online-snapshot/abort' 2023-07-22T12:35:23,015 DEBUG [RS:0;jenkins-hbase3:39575 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/2/online-snapshot/acquired' 2023-07-22T12:35:23,016 DEBUG [RS:0;jenkins-hbase3:39575 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-07-22T12:35:23,016 INFO [RS:0;jenkins-hbase3:39575 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-07-22T12:35:23,016 INFO [RS:0;jenkins-hbase3:39575 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-07-22T12:35:23,116 INFO [RS:0;jenkins-hbase3:39575 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-07-22T12:35:23,119 INFO [RS:0;jenkins-hbase3:39575 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C39575%2C1690029321362, suffix=, logDir=hdfs://localhost:38161/tmp/wal/WALs/jenkins-hbase3.apache.org,39575,1690029321362, archiveDir=hdfs://localhost:38161/tmp/wal/oldWALs, maxLogs=32 2023-07-22T12:35:23,136 DEBUG [RS:0;jenkins-hbase3:39575 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /tmp/wal/WALs/jenkins-hbase3.apache.org,39575,1690029321362/jenkins-hbase3.apache.org%2C39575%2C1690029321362.1690029323121, exclude list is [], retry=0 2023-07-22T12:35:23,143 DEBUG [RS-EventLoopGroup-6-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46725,DS-593ebd56-44b5-4a5f-a6ed-3e53df66d84b,DISK] 2023-07-22T12:35:23,151 INFO [RS:0;jenkins-hbase3:39575 {}] wal.AbstractFSWAL(832): New WAL /tmp/wal/WALs/jenkins-hbase3.apache.org,39575,1690029321362/jenkins-hbase3.apache.org%2C39575%2C1690029321362.1690029323121 2023-07-22T12:35:23,151 DEBUG [RS:0;jenkins-hbase3:39575 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:46725,DS-593ebd56-44b5-4a5f-a6ed-3e53df66d84b,DISK]] 2023-07-22T12:35:23,317 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-07-22T12:35:23,317 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a 2023-07-22T12:35:23,334 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:35:23,336 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-07-22T12:35:23,338 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-07-22T12:35:23,339 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:23,339 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:35:23,340 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-07-22T12:35:23,341 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-07-22T12:35:23,341 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:23,342 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:35:23,343 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-07-22T12:35:23,344 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-07-22T12:35:23,344 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:23,345 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:35:23,346 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/data/hbase/meta/1588230740 2023-07-22T12:35:23,347 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/data/hbase/meta/1588230740 2023-07-22T12:35:23,347 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38161/tmp/wal/data/hbase/meta/1588230740 2023-07-22T12:35:23,349 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-07-22T12:35:23,351 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-07-22T12:35:23,354 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38161/tmp/wal/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:35:23,355 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10491085760, jitterRate=-0.022941499948501587}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-07-22T12:35:23,355 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-07-22T12:35:23,355 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-07-22T12:35:23,355 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-07-22T12:35:23,355 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-07-22T12:35:23,355 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-07-22T12:35:23,355 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-07-22T12:35:23,356 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-07-22T12:35:23,356 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-07-22T12:35:23,357 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-07-22T12:35:23,357 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-07-22T12:35:23,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-07-22T12:35:23,359 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-07-22T12:35:23,361 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-07-22T12:35:23,392 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2023-07-22T12:35:23,512 DEBUG [jenkins-hbase3:46439 {}] assignment.AssignmentManager(2303): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-07-22T12:35:23,513 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,39575,1690029321362 2023-07-22T12:35:23,516 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,39575,1690029321362, state=OPENING 2023-07-22T12:35:23,517 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-07-22T12:35:23,520 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39575-0x100006222d70005, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-07-22T12:35:23,520 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-07-22T12:35:23,520 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /2/meta-region-server: CHANGED 2023-07-22T12:35:23,520 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /2/meta-region-server: CHANGED 2023-07-22T12:35:23,520 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,39575,1690029321362}] 2023-07-22T12:35:23,675 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(721): New admin connection to jenkins-hbase3.apache.org,39575,1690029321362 2023-07-22T12:35:23,676 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=AdminService, sasl=false 2023-07-22T12:35:23,679 INFO [RS-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:59388, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-07-22T12:35:23,683 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-07-22T12:35:23,683 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-07-22T12:35:23,684 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-07-22T12:35:23,687 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C39575%2C1690029321362.meta, suffix=.meta, logDir=hdfs://localhost:38161/tmp/wal/WALs/jenkins-hbase3.apache.org,39575,1690029321362, archiveDir=hdfs://localhost:38161/tmp/wal/oldWALs, maxLogs=32 2023-07-22T12:35:23,705 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /tmp/wal/WALs/jenkins-hbase3.apache.org,39575,1690029321362/jenkins-hbase3.apache.org%2C39575%2C1690029321362.meta.1690029323689.meta, exclude list is [], retry=0 2023-07-22T12:35:23,710 DEBUG [RS-EventLoopGroup-6-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37139,DS-e8126e2c-aa25-4919-967e-8d7c8d982603,DISK] 2023-07-22T12:35:23,713 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /tmp/wal/WALs/jenkins-hbase3.apache.org,39575,1690029321362/jenkins-hbase3.apache.org%2C39575%2C1690029321362.meta.1690029323689.meta 2023-07-22T12:35:23,713 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:37139,DS-e8126e2c-aa25-4919-967e-8d7c8d982603,DISK]] 2023-07-22T12:35:23,713 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:35:23,713 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:35:23,714 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-07-22T12:35:23,714 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-07-22T12:35:23,714 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-07-22T12:35:23,714 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-07-22T12:35:23,714 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:35:23,714 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-07-22T12:35:23,714 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-07-22T12:35:23,716 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-07-22T12:35:23,717 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-07-22T12:35:23,717 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:23,718 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:35:23,718 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-07-22T12:35:23,719 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-07-22T12:35:23,719 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:23,720 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:35:23,720 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-07-22T12:35:23,721 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-07-22T12:35:23,721 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:23,722 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:35:23,723 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/data/hbase/meta/1588230740 2023-07-22T12:35:23,724 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/data/hbase/meta/1588230740 2023-07-22T12:35:23,726 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38161/tmp/wal/data/hbase/meta/1588230740 2023-07-22T12:35:23,728 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-07-22T12:35:23,730 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-07-22T12:35:23,732 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9622985120, jitterRate=-0.10378967225551605}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-07-22T12:35:23,732 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-07-22T12:35:23,733 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2552): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1690029323675 2023-07-22T12:35:23,739 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2579): Finished post open deploy task for hbase:meta,,1.1588230740 2023-07-22T12:35:23,739 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-07-22T12:35:23,740 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,39575,1690029321362 2023-07-22T12:35:23,741 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,39575,1690029321362, state=OPEN 2023-07-22T12:35:23,743 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/2/meta-region-server 2023-07-22T12:35:23,743 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39575-0x100006222d70005, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/2/meta-region-server 2023-07-22T12:35:23,743 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /2/meta-region-server: CHANGED 2023-07-22T12:35:23,743 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /2/meta-region-server: CHANGED 2023-07-22T12:35:23,746 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=3, resume processing ppid=2 2023-07-22T12:35:23,746 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,39575,1690029321362 in 223 msec 2023-07-22T12:35:23,750 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=2, resume processing ppid=1 2023-07-22T12:35:23,750 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 389 msec 2023-07-22T12:35:23,753 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 888 msec 2023-07-22T12:35:23,753 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1062): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1690029323753, completionTime=-1 2023-07-22T12:35:23,753 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(830): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-07-22T12:35:23,753 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1629): Joining cluster... 2023-07-22T12:35:23,755 DEBUG [hconnection-0x1473a1c9-shared-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:35:23,757 INFO [RS-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:59390, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:35:23,759 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1641): Number of RegionServers=1 2023-07-22T12:35:23,759 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1690029383759 2023-07-22T12:35:23,759 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1690029443759 2023-07-22T12:35:23,759 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1648): Joined the cluster in 6 msec 2023-07-22T12:35:23,766 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,46439,1690029321204-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-07-22T12:35:23,766 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,46439,1690029321204-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-07-22T12:35:23,767 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,46439,1690029321204-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-07-22T12:35:23,767 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:46439, period=300000, unit=MILLISECONDS is enabled. 2023-07-22T12:35:23,767 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-07-22T12:35:23,767 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-07-22T12:35:23,767 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2367): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-07-22T12:35:23,768 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-07-22T12:35:23,769 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-07-22T12:35:23,770 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:35:23,771 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:23,772 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:35:24,186 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 88bacc909086c649c0d4264069cc9481, NAME => 'hbase:namespace,,1690029323767.88bacc909086c649c0d4264069cc9481.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a 2023-07-22T12:35:24,599 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1690029323767.88bacc909086c649c0d4264069cc9481.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:35:24,599 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing 88bacc909086c649c0d4264069cc9481, disabling compactions & flushes 2023-07-22T12:35:24,600 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1690029323767.88bacc909086c649c0d4264069cc9481. 2023-07-22T12:35:24,600 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1690029323767.88bacc909086c649c0d4264069cc9481. 2023-07-22T12:35:24,600 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1690029323767.88bacc909086c649c0d4264069cc9481. after waiting 0 ms 2023-07-22T12:35:24,600 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1690029323767.88bacc909086c649c0d4264069cc9481. 2023-07-22T12:35:24,600 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1690029323767.88bacc909086c649c0d4264069cc9481. 2023-07-22T12:35:24,600 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for 88bacc909086c649c0d4264069cc9481: 2023-07-22T12:35:24,602 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:35:24,603 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1690029323767.88bacc909086c649c0d4264069cc9481.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1690029324602"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029324602"}]},"ts":"1690029324602"} 2023-07-22T12:35:24,624 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:35:24,625 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:35:24,626 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029324625"}]},"ts":"1690029324625"} 2023-07-22T12:35:24,630 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-07-22T12:35:24,635 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=88bacc909086c649c0d4264069cc9481, ASSIGN}] 2023-07-22T12:35:24,638 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=88bacc909086c649c0d4264069cc9481, ASSIGN 2023-07-22T12:35:24,640 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=88bacc909086c649c0d4264069cc9481, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,39575,1690029321362; forceNewPlan=false, retain=false 2023-07-22T12:35:24,791 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=88bacc909086c649c0d4264069cc9481, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,39575,1690029321362 2023-07-22T12:35:24,795 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 88bacc909086c649c0d4264069cc9481, server=jenkins-hbase3.apache.org,39575,1690029321362}] 2023-07-22T12:35:24,955 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1690029323767.88bacc909086c649c0d4264069cc9481. 2023-07-22T12:35:24,956 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => 88bacc909086c649c0d4264069cc9481, NAME => 'hbase:namespace,,1690029323767.88bacc909086c649c0d4264069cc9481.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:35:24,956 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:35:24,956 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 88bacc909086c649c0d4264069cc9481 2023-07-22T12:35:24,956 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1690029323767.88bacc909086c649c0d4264069cc9481.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:35:24,956 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for 88bacc909086c649c0d4264069cc9481 2023-07-22T12:35:24,957 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for 88bacc909086c649c0d4264069cc9481 2023-07-22T12:35:24,958 INFO [StoreOpener-88bacc909086c649c0d4264069cc9481-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 88bacc909086c649c0d4264069cc9481 2023-07-22T12:35:24,960 INFO [StoreOpener-88bacc909086c649c0d4264069cc9481-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88bacc909086c649c0d4264069cc9481 columnFamilyName info 2023-07-22T12:35:24,960 DEBUG [StoreOpener-88bacc909086c649c0d4264069cc9481-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:24,961 INFO [StoreOpener-88bacc909086c649c0d4264069cc9481-1 {}] regionserver.HStore(324): Store=88bacc909086c649c0d4264069cc9481/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:35:24,962 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/data/hbase/namespace/88bacc909086c649c0d4264069cc9481 2023-07-22T12:35:24,963 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/data/hbase/namespace/88bacc909086c649c0d4264069cc9481 2023-07-22T12:35:24,963 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38161/tmp/wal/data/hbase/namespace/88bacc909086c649c0d4264069cc9481 2023-07-22T12:35:24,966 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for 88bacc909086c649c0d4264069cc9481 2023-07-22T12:35:24,969 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38161/tmp/wal/data/hbase/namespace/88bacc909086c649c0d4264069cc9481/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:35:24,970 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened 88bacc909086c649c0d4264069cc9481; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11837945760, jitterRate=0.10249461233615875}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:35:24,970 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for 88bacc909086c649c0d4264069cc9481: 2023-07-22T12:35:24,971 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2552): Post open deploy tasks for hbase:namespace,,1690029323767.88bacc909086c649c0d4264069cc9481., pid=6, masterSystemTime=1690029324948 2023-07-22T12:35:24,974 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2579): Finished post open deploy task for hbase:namespace,,1690029323767.88bacc909086c649c0d4264069cc9481. 2023-07-22T12:35:24,974 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1690029323767.88bacc909086c649c0d4264069cc9481. 2023-07-22T12:35:24,975 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=88bacc909086c649c0d4264069cc9481, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,39575,1690029321362 2023-07-22T12:35:24,981 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=6, resume processing ppid=5 2023-07-22T12:35:24,981 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 88bacc909086c649c0d4264069cc9481, server=jenkins-hbase3.apache.org,39575,1690029321362 in 183 msec 2023-07-22T12:35:24,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=5, resume processing ppid=4 2023-07-22T12:35:24,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=88bacc909086c649c0d4264069cc9481, ASSIGN in 346 msec 2023-07-22T12:35:24,986 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:35:24,986 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029324986"}]},"ts":"1690029324986"} 2023-07-22T12:35:24,989 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-07-22T12:35:24,992 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:35:24,995 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 1.2250 sec 2023-07-22T12:35:25,071 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Set watcher on znode that does not yet exist, /2/namespace 2023-07-22T12:35:25,072 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/2/namespace 2023-07-22T12:35:25,072 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39575-0x100006222d70005, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-07-22T12:35:25,072 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-07-22T12:35:25,078 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-07-22T12:35:25,090 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/namespace 2023-07-22T12:35:25,095 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 16 msec 2023-07-22T12:35:25,100 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-07-22T12:35:25,108 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/namespace 2023-07-22T12:35:25,112 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 12 msec 2023-07-22T12:35:25,125 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/2/namespace/default 2023-07-22T12:35:25,127 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/2/namespace/hbase 2023-07-22T12:35:25,127 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1192): Master has completed initialization 3.695sec 2023-07-22T12:35:25,127 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-07-22T12:35:25,127 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-07-22T12:35:25,127 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-07-22T12:35:25,127 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-07-22T12:35:25,127 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-07-22T12:35:25,127 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,46439,1690029321204-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-07-22T12:35:25,127 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,46439,1690029321204-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-07-22T12:35:25,129 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1295): Balancer post startup initialization complete, took 0 seconds 2023-07-22T12:35:25,129 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-07-22T12:35:25,219 DEBUG [Listener at localhost/42321 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x431071ca to 127.0.0.1:54609 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:35:25,223 DEBUG [Listener at localhost/42321 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5988aef7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:35:25,225 DEBUG [hconnection-0x77b967bb-shared-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:35:25,227 INFO [RS-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:59394, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:35:25,230 INFO [Listener at localhost/42321 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,46439,1690029321204 2023-07-22T12:35:25,230 INFO [Listener at localhost/42321 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-07-22T12:35:25,230 INFO [Listener at localhost/42321 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/test.cache.data in system properties and HBase conf 2023-07-22T12:35:25,230 INFO [Listener at localhost/42321 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/hadoop.tmp.dir in system properties and HBase conf 2023-07-22T12:35:25,230 INFO [Listener at localhost/42321 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/hadoop.log.dir in system properties and HBase conf 2023-07-22T12:35:25,230 INFO [Listener at localhost/42321 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/mapreduce.cluster.local.dir in system properties and HBase conf 2023-07-22T12:35:25,230 INFO [Listener at localhost/42321 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-07-22T12:35:25,230 INFO [Listener at localhost/42321 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-07-22T12:35:25,230 INFO [Listener at localhost/42321 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-07-22T12:35:25,230 INFO [Listener at localhost/42321 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-07-22T12:35:25,230 INFO [Listener at localhost/42321 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-07-22T12:35:25,230 INFO [Listener at localhost/42321 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-07-22T12:35:25,230 INFO [Listener at localhost/42321 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-07-22T12:35:25,231 INFO [Listener at localhost/42321 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-07-22T12:35:25,231 INFO [Listener at localhost/42321 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-07-22T12:35:25,231 INFO [Listener at localhost/42321 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/dfs.journalnode.edits.dir in system properties and HBase conf 2023-07-22T12:35:25,231 INFO [Listener at localhost/42321 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-07-22T12:35:25,231 INFO [Listener at localhost/42321 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/nfs.dump.dir in system properties and HBase conf 2023-07-22T12:35:25,231 INFO [Listener at localhost/42321 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/java.io.tmpdir in system properties and HBase conf 2023-07-22T12:35:25,231 INFO [Listener at localhost/42321 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/dfs.journalnode.edits.dir in system properties and HBase conf 2023-07-22T12:35:25,231 INFO [Listener at localhost/42321 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-07-22T12:35:25,231 INFO [Listener at localhost/42321 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-07-22T12:35:27,231 WARN [Thread-467 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:35:27,581 INFO [Thread-467 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:35:27,587 INFO [Thread-467 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_40887_jobhistory____hizih1/webapp 2023-07-22T12:35:27,760 INFO [Thread-467 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-07-22T12:35:27,760 INFO [Thread-467 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-07-22T12:35:27,760 INFO [Thread-467 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-07-22T12:35:27,763 INFO [Thread-467 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-07-22T12:35:27,847 INFO [Thread-467 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:35:27,956 INFO [Thread-467 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:35:28,280 INFO [Thread-467 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-07-22T12:35:28,327 INFO [Thread-467 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:40887 2023-07-22T12:35:28,733 WARN [Listener at jenkins-hbase3.apache.org/40021 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:35:28,828 WARN [Listener at jenkins-hbase3.apache.org/40021 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-07-22T12:35:28,829 INFO [Listener at jenkins-hbase3.apache.org/40021 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:35:28,835 INFO [Listener at jenkins-hbase3.apache.org/40021 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_33425_cluster____658xws/webapp 2023-07-22T12:35:28,919 INFO [Listener at jenkins-hbase3.apache.org/40021 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-07-22T12:35:28,919 INFO [Listener at jenkins-hbase3.apache.org/40021 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-07-22T12:35:28,919 INFO [Listener at jenkins-hbase3.apache.org/40021 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-07-22T12:35:28,919 INFO [Listener at jenkins-hbase3.apache.org/40021 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-07-22T12:35:28,928 INFO [Listener at jenkins-hbase3.apache.org/40021 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:35:29,149 INFO [Listener at jenkins-hbase3.apache.org/40021 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:35:29,228 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:35:29,287 INFO [Listener at jenkins-hbase3.apache.org/40021 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:35:29,296 INFO [Listener at jenkins-hbase3.apache.org/40021 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:33425 2023-07-22T12:35:29,724 WARN [Listener at jenkins-hbase3.apache.org/43955 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:35:29,750 WARN [Listener at jenkins-hbase3.apache.org/43955 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-07-22T12:35:29,751 INFO [Listener at jenkins-hbase3.apache.org/43955 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:35:29,757 INFO [Listener at jenkins-hbase3.apache.org/43955 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_44705_node____.ihpa05/webapp 2023-07-22T12:35:29,808 INFO [Listener at jenkins-hbase3.apache.org/43955 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-07-22T12:35:29,808 INFO [Listener at jenkins-hbase3.apache.org/43955 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-07-22T12:35:29,808 INFO [Listener at jenkins-hbase3.apache.org/43955 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-07-22T12:35:29,808 INFO [Listener at jenkins-hbase3.apache.org/43955 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-07-22T12:35:29,816 INFO [Listener at jenkins-hbase3.apache.org/43955 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:35:29,839 INFO [Listener at jenkins-hbase3.apache.org/43955 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:35:29,938 INFO [Listener at jenkins-hbase3.apache.org/43955 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:35:29,952 INFO [Listener at jenkins-hbase3.apache.org/43955 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:44705 2023-07-22T12:35:30,244 WARN [Listener at jenkins-hbase3.apache.org/44413 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:35:30,253 WARN [Listener at jenkins-hbase3.apache.org/44413 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-07-22T12:35:30,254 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:35:30,262 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_45295_node____vyc6h8/webapp 2023-07-22T12:35:30,338 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-07-22T12:35:30,339 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-07-22T12:35:30,339 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-07-22T12:35:30,339 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-07-22T12:35:30,350 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:35:30,375 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:35:30,478 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:35:30,500 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:45295 2023-07-22T12:35:30,551 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-07-22T12:35:30,551 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] backup.TestBackupBase(301): ROOTDIR hdfs://localhost:38685/backupUT 2023-07-22T12:35:30,567 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] backup.TestBackupBase(306): REMOTE ROOTDIR hdfs://localhost:38161/backupUT 2023-07-22T12:35:30,584 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:35:30,590 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:41848, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:35:30,603 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns1'} 2023-07-22T12:35:30,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] procedure2.ProcedureExecutor(1033): Stored pid=9, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns1 2023-07-22T12:35:30,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=9 2023-07-22T12:35:30,634 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:35:30,639 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=9, state=SUCCESS; CreateNamespaceProcedure, namespace=ns1 in 33 msec 2023-07-22T12:35:30,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=9 2023-07-22T12:35:30,731 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns2'} 2023-07-22T12:35:30,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] procedure2.ProcedureExecutor(1033): Stored pid=10, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns2 2023-07-22T12:35:30,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=10 2023-07-22T12:35:30,746 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:35:30,752 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=10, state=SUCCESS; CreateNamespaceProcedure, namespace=ns2 in 19 msec 2023-07-22T12:35:30,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=10 2023-07-22T12:35:30,843 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns3'} 2023-07-22T12:35:30,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] procedure2.ProcedureExecutor(1033): Stored pid=11, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns3 2023-07-22T12:35:30,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=11 2023-07-22T12:35:30,856 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:35:30,863 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=11, state=SUCCESS; CreateNamespaceProcedure, namespace=ns3 in 17 msec 2023-07-22T12:35:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=11 2023-07-22T12:35:30,954 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns4'} 2023-07-22T12:35:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] procedure2.ProcedureExecutor(1033): Stored pid=12, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns4 2023-07-22T12:35:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=12 2023-07-22T12:35:30,977 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:35:30,991 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=12, state=SUCCESS; CreateNamespaceProcedure, namespace=ns4 in 34 msec 2023-07-22T12:35:31,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=12 2023-07-22T12:35:31,082 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'test-1690029330567', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:35:31,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] procedure2.ProcedureExecutor(1033): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=test-1690029330567 2023-07-22T12:35:31,089 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=test-1690029330567 execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:35:31,089 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "test-1690029330567" procId is: 13 2023-07-22T12:35:31,090 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:31,091 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=test-1690029330567 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:35:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-07-22T12:35:31,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-07-22T12:35:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-07-22T12:35:31,520 INFO [RegionOpenAndInit-test-1690029330567-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => e2f9453920ffaa2bfd87a3743468e74e, NAME => 'test-1690029330567,,1690029331075.e2f9453920ffaa2bfd87a3743468e74e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1690029330567', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f 2023-07-22T12:35:31,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-07-22T12:35:31,933 DEBUG [RegionOpenAndInit-test-1690029330567-pool-0 {}] regionserver.HRegion(888): Instantiated test-1690029330567,,1690029331075.e2f9453920ffaa2bfd87a3743468e74e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:35:31,934 DEBUG [RegionOpenAndInit-test-1690029330567-pool-0 {}] regionserver.HRegion(1668): Closing e2f9453920ffaa2bfd87a3743468e74e, disabling compactions & flushes 2023-07-22T12:35:31,934 INFO [RegionOpenAndInit-test-1690029330567-pool-0 {}] regionserver.HRegion(1690): Closing region test-1690029330567,,1690029331075.e2f9453920ffaa2bfd87a3743468e74e. 2023-07-22T12:35:31,934 DEBUG [RegionOpenAndInit-test-1690029330567-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1690029330567,,1690029331075.e2f9453920ffaa2bfd87a3743468e74e. 2023-07-22T12:35:31,934 DEBUG [RegionOpenAndInit-test-1690029330567-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1690029330567,,1690029331075.e2f9453920ffaa2bfd87a3743468e74e. after waiting 0 ms 2023-07-22T12:35:31,934 DEBUG [RegionOpenAndInit-test-1690029330567-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1690029330567,,1690029331075.e2f9453920ffaa2bfd87a3743468e74e. 2023-07-22T12:35:31,934 INFO [RegionOpenAndInit-test-1690029330567-pool-0 {}] regionserver.HRegion(1909): Closed test-1690029330567,,1690029331075.e2f9453920ffaa2bfd87a3743468e74e. 2023-07-22T12:35:31,934 DEBUG [RegionOpenAndInit-test-1690029330567-pool-0 {}] regionserver.HRegion(1622): Region close journal for e2f9453920ffaa2bfd87a3743468e74e: 2023-07-22T12:35:31,936 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=test-1690029330567 execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:35:31,937 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1690029330567,,1690029331075.e2f9453920ffaa2bfd87a3743468e74e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1690029331937"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029331937"}]},"ts":"1690029331937"} 2023-07-22T12:35:31,940 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:35:31,941 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=test-1690029330567 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:35:31,941 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1690029330567","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029331941"}]},"ts":"1690029331941"} 2023-07-22T12:35:31,944 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1690029330567, state=ENABLING in hbase:meta 2023-07-22T12:35:31,947 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1690029330567, region=e2f9453920ffaa2bfd87a3743468e74e, ASSIGN}] 2023-07-22T12:35:31,950 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1690029330567, region=e2f9453920ffaa2bfd87a3743468e74e, ASSIGN 2023-07-22T12:35:31,952 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1690029330567, region=e2f9453920ffaa2bfd87a3743468e74e, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38147,1690029314593; forceNewPlan=false, retain=false 2023-07-22T12:35:32,104 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=e2f9453920ffaa2bfd87a3743468e74e, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:32,107 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; OpenRegionProcedure e2f9453920ffaa2bfd87a3743468e74e, server=jenkins-hbase3.apache.org,38147,1690029314593}] 2023-07-22T12:35:32,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-07-22T12:35:32,271 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(130): Open test-1690029330567,,1690029331075.e2f9453920ffaa2bfd87a3743468e74e. 2023-07-22T12:35:32,271 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7254): Opening region: {ENCODED => e2f9453920ffaa2bfd87a3743468e74e, NAME => 'test-1690029330567,,1690029331075.e2f9453920ffaa2bfd87a3743468e74e.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:35:32,271 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:35:32,272 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1690029330567 e2f9453920ffaa2bfd87a3743468e74e 2023-07-22T12:35:32,272 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(888): Instantiated test-1690029330567,,1690029331075.e2f9453920ffaa2bfd87a3743468e74e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:35:32,272 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7296): checking encryption for e2f9453920ffaa2bfd87a3743468e74e 2023-07-22T12:35:32,272 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7299): checking classloading for e2f9453920ffaa2bfd87a3743468e74e 2023-07-22T12:35:32,274 INFO [StoreOpener-e2f9453920ffaa2bfd87a3743468e74e-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region e2f9453920ffaa2bfd87a3743468e74e 2023-07-22T12:35:32,276 INFO [StoreOpener-e2f9453920ffaa2bfd87a3743468e74e-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2f9453920ffaa2bfd87a3743468e74e columnFamilyName f 2023-07-22T12:35:32,276 DEBUG [StoreOpener-e2f9453920ffaa2bfd87a3743468e74e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:32,277 INFO [StoreOpener-e2f9453920ffaa2bfd87a3743468e74e-1 {}] regionserver.HStore(324): Store=e2f9453920ffaa2bfd87a3743468e74e/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:35:32,278 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/test-1690029330567/e2f9453920ffaa2bfd87a3743468e74e 2023-07-22T12:35:32,278 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/test-1690029330567/e2f9453920ffaa2bfd87a3743468e74e 2023-07-22T12:35:32,282 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1076): writing seq id for e2f9453920ffaa2bfd87a3743468e74e 2023-07-22T12:35:32,285 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/test-1690029330567/e2f9453920ffaa2bfd87a3743468e74e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:35:32,286 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): Opened e2f9453920ffaa2bfd87a3743468e74e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11107563360, jitterRate=0.034472450613975525}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:35:32,287 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(992): Region open journal for e2f9453920ffaa2bfd87a3743468e74e: 2023-07-22T12:35:32,288 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2552): Post open deploy tasks for test-1690029330567,,1690029331075.e2f9453920ffaa2bfd87a3743468e74e., pid=15, masterSystemTime=1690029332260 2023-07-22T12:35:32,291 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2579): Finished post open deploy task for test-1690029330567,,1690029331075.e2f9453920ffaa2bfd87a3743468e74e. 2023-07-22T12:35:32,291 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(158): Opened test-1690029330567,,1690029331075.e2f9453920ffaa2bfd87a3743468e74e. 2023-07-22T12:35:32,291 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=e2f9453920ffaa2bfd87a3743468e74e, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:32,296 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=15, resume processing ppid=14 2023-07-22T12:35:32,296 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=15, ppid=14, state=SUCCESS; OpenRegionProcedure e2f9453920ffaa2bfd87a3743468e74e, server=jenkins-hbase3.apache.org,38147,1690029314593 in 187 msec 2023-07-22T12:35:32,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=14, resume processing ppid=13 2023-07-22T12:35:32,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=14, ppid=13, state=SUCCESS; TransitRegionStateProcedure table=test-1690029330567, region=e2f9453920ffaa2bfd87a3743468e74e, ASSIGN in 349 msec 2023-07-22T12:35:32,300 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=test-1690029330567 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:35:32,300 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1690029330567","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029332300"}]},"ts":"1690029332300"} 2023-07-22T12:35:32,301 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1690029330567, state=ENABLED in hbase:meta 2023-07-22T12:35:32,304 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=test-1690029330567 execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:35:32,305 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=13, state=SUCCESS; CreateTableProcedure table=test-1690029330567 in 1.2220 sec 2023-07-22T12:35:33,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-07-22T12:35:33,208 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: default:test-1690029330567, procId: 13 completed 2023-07-22T12:35:33,208 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x41cc3954 to 127.0.0.1:54609 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:35:33,215 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bc15c84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:35:33,217 DEBUG [hconnection-0x45cc8d21-metaLookup-shared--pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:35:33,220 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:43398, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:35:33,228 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] regionserver.HRegion(8210): writing data to region test-1690029330567,,1690029331075.e2f9453920ffaa2bfd87a3743468e74e. with WAL disabled. Data may be lost in the event of a crash. 2023-07-22T12:35:33,314 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns2:test-16900293305671', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:35:33,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] procedure2.ProcedureExecutor(1033): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:test-16900293305671 2023-07-22T12:35:33,317 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16900293305671 execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:35:33,318 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:33,318 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "test-16900293305671" procId is: 16 2023-07-22T12:35:33,319 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:test-16900293305671 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:35:33,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-07-22T12:35:33,332 INFO [RegionOpenAndInit-ns2:test-16900293305671-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 8f674da8b67afb2bfe7cd3f65c30fe3f, NAME => 'ns2:test-16900293305671,,1690029333313.8f674da8b67afb2bfe7cd3f65c30fe3f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16900293305671', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f 2023-07-22T12:35:33,343 DEBUG [RegionOpenAndInit-ns2:test-16900293305671-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16900293305671,,1690029333313.8f674da8b67afb2bfe7cd3f65c30fe3f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:35:33,343 DEBUG [RegionOpenAndInit-ns2:test-16900293305671-pool-0 {}] regionserver.HRegion(1668): Closing 8f674da8b67afb2bfe7cd3f65c30fe3f, disabling compactions & flushes 2023-07-22T12:35:33,343 INFO [RegionOpenAndInit-ns2:test-16900293305671-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16900293305671,,1690029333313.8f674da8b67afb2bfe7cd3f65c30fe3f. 2023-07-22T12:35:33,343 DEBUG [RegionOpenAndInit-ns2:test-16900293305671-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16900293305671,,1690029333313.8f674da8b67afb2bfe7cd3f65c30fe3f. 2023-07-22T12:35:33,343 DEBUG [RegionOpenAndInit-ns2:test-16900293305671-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16900293305671,,1690029333313.8f674da8b67afb2bfe7cd3f65c30fe3f. after waiting 0 ms 2023-07-22T12:35:33,343 DEBUG [RegionOpenAndInit-ns2:test-16900293305671-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16900293305671,,1690029333313.8f674da8b67afb2bfe7cd3f65c30fe3f. 2023-07-22T12:35:33,343 INFO [RegionOpenAndInit-ns2:test-16900293305671-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16900293305671,,1690029333313.8f674da8b67afb2bfe7cd3f65c30fe3f. 2023-07-22T12:35:33,343 DEBUG [RegionOpenAndInit-ns2:test-16900293305671-pool-0 {}] regionserver.HRegion(1622): Region close journal for 8f674da8b67afb2bfe7cd3f65c30fe3f: 2023-07-22T12:35:33,345 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:test-16900293305671 execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:35:33,345 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16900293305671,,1690029333313.8f674da8b67afb2bfe7cd3f65c30fe3f.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1690029333345"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029333345"}]},"ts":"1690029333345"} 2023-07-22T12:35:33,347 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:35:33,348 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:test-16900293305671 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:35:33,348 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16900293305671","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029333348"}]},"ts":"1690029333348"} 2023-07-22T12:35:33,350 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16900293305671, state=ENABLING in hbase:meta 2023-07-22T12:35:33,354 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16900293305671, region=8f674da8b67afb2bfe7cd3f65c30fe3f, ASSIGN}] 2023-07-22T12:35:33,356 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16900293305671, region=8f674da8b67afb2bfe7cd3f65c30fe3f, ASSIGN 2023-07-22T12:35:33,357 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16900293305671, region=8f674da8b67afb2bfe7cd3f65c30fe3f, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38147,1690029314593; forceNewPlan=false, retain=false 2023-07-22T12:35:33,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-07-22T12:35:33,509 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=8f674da8b67afb2bfe7cd3f65c30fe3f, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:33,511 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; OpenRegionProcedure 8f674da8b67afb2bfe7cd3f65c30fe3f, server=jenkins-hbase3.apache.org,38147,1690029314593}] 2023-07-22T12:35:33,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-07-22T12:35:33,667 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(130): Open ns2:test-16900293305671,,1690029333313.8f674da8b67afb2bfe7cd3f65c30fe3f. 2023-07-22T12:35:33,668 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7254): Opening region: {ENCODED => 8f674da8b67afb2bfe7cd3f65c30fe3f, NAME => 'ns2:test-16900293305671,,1690029333313.8f674da8b67afb2bfe7cd3f65c30fe3f.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:35:33,668 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:35:33,669 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16900293305671 8f674da8b67afb2bfe7cd3f65c30fe3f 2023-07-22T12:35:33,669 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(888): Instantiated ns2:test-16900293305671,,1690029333313.8f674da8b67afb2bfe7cd3f65c30fe3f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:35:33,669 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7296): checking encryption for 8f674da8b67afb2bfe7cd3f65c30fe3f 2023-07-22T12:35:33,669 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7299): checking classloading for 8f674da8b67afb2bfe7cd3f65c30fe3f 2023-07-22T12:35:33,671 INFO [StoreOpener-8f674da8b67afb2bfe7cd3f65c30fe3f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 8f674da8b67afb2bfe7cd3f65c30fe3f 2023-07-22T12:35:33,672 INFO [StoreOpener-8f674da8b67afb2bfe7cd3f65c30fe3f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f674da8b67afb2bfe7cd3f65c30fe3f columnFamilyName f 2023-07-22T12:35:33,672 DEBUG [StoreOpener-8f674da8b67afb2bfe7cd3f65c30fe3f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:33,673 INFO [StoreOpener-8f674da8b67afb2bfe7cd3f65c30fe3f-1 {}] regionserver.HStore(324): Store=8f674da8b67afb2bfe7cd3f65c30fe3f/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:35:33,674 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/ns2/test-16900293305671/8f674da8b67afb2bfe7cd3f65c30fe3f 2023-07-22T12:35:33,675 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/ns2/test-16900293305671/8f674da8b67afb2bfe7cd3f65c30fe3f 2023-07-22T12:35:33,677 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1076): writing seq id for 8f674da8b67afb2bfe7cd3f65c30fe3f 2023-07-22T12:35:33,680 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/ns2/test-16900293305671/8f674da8b67afb2bfe7cd3f65c30fe3f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:35:33,681 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): Opened 8f674da8b67afb2bfe7cd3f65c30fe3f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10734588640, jitterRate=-2.635270357131958E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:35:33,681 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(992): Region open journal for 8f674da8b67afb2bfe7cd3f65c30fe3f: 2023-07-22T12:35:33,683 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2552): Post open deploy tasks for ns2:test-16900293305671,,1690029333313.8f674da8b67afb2bfe7cd3f65c30fe3f., pid=18, masterSystemTime=1690029333663 2023-07-22T12:35:33,685 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2579): Finished post open deploy task for ns2:test-16900293305671,,1690029333313.8f674da8b67afb2bfe7cd3f65c30fe3f. 2023-07-22T12:35:33,685 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(158): Opened ns2:test-16900293305671,,1690029333313.8f674da8b67afb2bfe7cd3f65c30fe3f. 2023-07-22T12:35:33,685 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=8f674da8b67afb2bfe7cd3f65c30fe3f, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:33,692 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=18, resume processing ppid=17 2023-07-22T12:35:33,692 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=18, ppid=17, state=SUCCESS; OpenRegionProcedure 8f674da8b67afb2bfe7cd3f65c30fe3f, server=jenkins-hbase3.apache.org,38147,1690029314593 in 178 msec 2023-07-22T12:35:33,694 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=17, resume processing ppid=16 2023-07-22T12:35:33,694 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=17, ppid=16, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16900293305671, region=8f674da8b67afb2bfe7cd3f65c30fe3f, ASSIGN in 338 msec 2023-07-22T12:35:33,695 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:test-16900293305671 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:35:33,695 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16900293305671","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029333695"}]},"ts":"1690029333695"} 2023-07-22T12:35:33,697 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16900293305671, state=ENABLED in hbase:meta 2023-07-22T12:35:33,700 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16900293305671 execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:35:33,701 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=16, state=SUCCESS; CreateTableProcedure table=ns2:test-16900293305671 in 386 msec 2023-07-22T12:35:33,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-07-22T12:35:33,924 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: ns2:test-16900293305671, procId: 16 completed 2023-07-22T12:35:33,930 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] regionserver.HRegion(8210): writing data to region ns2:test-16900293305671,,1690029333313.8f674da8b67afb2bfe7cd3f65c30fe3f. with WAL disabled. Data may be lost in the event of a crash. 2023-07-22T12:35:34,007 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns3:test-16900293305672', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:35:34,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] procedure2.ProcedureExecutor(1033): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:test-16900293305672 2023-07-22T12:35:34,011 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16900293305672 execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:35:34,011 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:34,011 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "test-16900293305672" procId is: 19 2023-07-22T12:35:34,012 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:test-16900293305672 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:35:34,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-07-22T12:35:34,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-07-22T12:35:34,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-07-22T12:35:34,427 INFO [RegionOpenAndInit-ns3:test-16900293305672-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 30c77692ccc4e0df458e1f12cc34f3f8, NAME => 'ns3:test-16900293305672,,1690029334007.30c77692ccc4e0df458e1f12cc34f3f8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16900293305672', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f 2023-07-22T12:35:34,536 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-16900293305671' 2023-07-22T12:35:34,537 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1690029330567' 2023-07-22T12:35:34,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-07-22T12:35:34,844 DEBUG [RegionOpenAndInit-ns3:test-16900293305672-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16900293305672,,1690029334007.30c77692ccc4e0df458e1f12cc34f3f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:35:34,844 DEBUG [RegionOpenAndInit-ns3:test-16900293305672-pool-0 {}] regionserver.HRegion(1668): Closing 30c77692ccc4e0df458e1f12cc34f3f8, disabling compactions & flushes 2023-07-22T12:35:34,844 INFO [RegionOpenAndInit-ns3:test-16900293305672-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16900293305672,,1690029334007.30c77692ccc4e0df458e1f12cc34f3f8. 2023-07-22T12:35:34,844 DEBUG [RegionOpenAndInit-ns3:test-16900293305672-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16900293305672,,1690029334007.30c77692ccc4e0df458e1f12cc34f3f8. 2023-07-22T12:35:34,844 DEBUG [RegionOpenAndInit-ns3:test-16900293305672-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16900293305672,,1690029334007.30c77692ccc4e0df458e1f12cc34f3f8. after waiting 0 ms 2023-07-22T12:35:34,844 DEBUG [RegionOpenAndInit-ns3:test-16900293305672-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16900293305672,,1690029334007.30c77692ccc4e0df458e1f12cc34f3f8. 2023-07-22T12:35:34,844 INFO [RegionOpenAndInit-ns3:test-16900293305672-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16900293305672,,1690029334007.30c77692ccc4e0df458e1f12cc34f3f8. 2023-07-22T12:35:34,844 DEBUG [RegionOpenAndInit-ns3:test-16900293305672-pool-0 {}] regionserver.HRegion(1622): Region close journal for 30c77692ccc4e0df458e1f12cc34f3f8: 2023-07-22T12:35:34,846 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:test-16900293305672 execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:35:34,846 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16900293305672,,1690029334007.30c77692ccc4e0df458e1f12cc34f3f8.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1690029334846"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029334846"}]},"ts":"1690029334846"} 2023-07-22T12:35:34,848 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:35:34,849 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:test-16900293305672 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:35:34,850 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16900293305672","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029334849"}]},"ts":"1690029334849"} 2023-07-22T12:35:34,851 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16900293305672, state=ENABLING in hbase:meta 2023-07-22T12:35:34,855 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16900293305672, region=30c77692ccc4e0df458e1f12cc34f3f8, ASSIGN}] 2023-07-22T12:35:34,858 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16900293305672, region=30c77692ccc4e0df458e1f12cc34f3f8, ASSIGN 2023-07-22T12:35:34,859 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16900293305672, region=30c77692ccc4e0df458e1f12cc34f3f8, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38147,1690029314593; forceNewPlan=false, retain=false 2023-07-22T12:35:35,011 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=30c77692ccc4e0df458e1f12cc34f3f8, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:35,014 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; OpenRegionProcedure 30c77692ccc4e0df458e1f12cc34f3f8, server=jenkins-hbase3.apache.org,38147,1690029314593}] 2023-07-22T12:35:35,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-07-22T12:35:35,171 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(130): Open ns3:test-16900293305672,,1690029334007.30c77692ccc4e0df458e1f12cc34f3f8. 2023-07-22T12:35:35,171 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7254): Opening region: {ENCODED => 30c77692ccc4e0df458e1f12cc34f3f8, NAME => 'ns3:test-16900293305672,,1690029334007.30c77692ccc4e0df458e1f12cc34f3f8.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:35:35,171 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:35:35,171 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16900293305672 30c77692ccc4e0df458e1f12cc34f3f8 2023-07-22T12:35:35,171 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(888): Instantiated ns3:test-16900293305672,,1690029334007.30c77692ccc4e0df458e1f12cc34f3f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:35:35,171 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7296): checking encryption for 30c77692ccc4e0df458e1f12cc34f3f8 2023-07-22T12:35:35,172 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7299): checking classloading for 30c77692ccc4e0df458e1f12cc34f3f8 2023-07-22T12:35:35,173 INFO [StoreOpener-30c77692ccc4e0df458e1f12cc34f3f8-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 30c77692ccc4e0df458e1f12cc34f3f8 2023-07-22T12:35:35,175 INFO [StoreOpener-30c77692ccc4e0df458e1f12cc34f3f8-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30c77692ccc4e0df458e1f12cc34f3f8 columnFamilyName f 2023-07-22T12:35:35,175 DEBUG [StoreOpener-30c77692ccc4e0df458e1f12cc34f3f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:35,176 INFO [StoreOpener-30c77692ccc4e0df458e1f12cc34f3f8-1 {}] regionserver.HStore(324): Store=30c77692ccc4e0df458e1f12cc34f3f8/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:35:35,177 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/ns3/test-16900293305672/30c77692ccc4e0df458e1f12cc34f3f8 2023-07-22T12:35:35,178 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/ns3/test-16900293305672/30c77692ccc4e0df458e1f12cc34f3f8 2023-07-22T12:35:35,181 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1076): writing seq id for 30c77692ccc4e0df458e1f12cc34f3f8 2023-07-22T12:35:35,184 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/ns3/test-16900293305672/30c77692ccc4e0df458e1f12cc34f3f8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:35:35,185 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): Opened 30c77692ccc4e0df458e1f12cc34f3f8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11336505920, jitterRate=0.05579438805580139}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:35:35,186 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(992): Region open journal for 30c77692ccc4e0df458e1f12cc34f3f8: 2023-07-22T12:35:35,187 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2552): Post open deploy tasks for ns3:test-16900293305672,,1690029334007.30c77692ccc4e0df458e1f12cc34f3f8., pid=21, masterSystemTime=1690029335166 2023-07-22T12:35:35,189 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2579): Finished post open deploy task for ns3:test-16900293305672,,1690029334007.30c77692ccc4e0df458e1f12cc34f3f8. 2023-07-22T12:35:35,189 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(158): Opened ns3:test-16900293305672,,1690029334007.30c77692ccc4e0df458e1f12cc34f3f8. 2023-07-22T12:35:35,189 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=30c77692ccc4e0df458e1f12cc34f3f8, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:35,194 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=21, resume processing ppid=20 2023-07-22T12:35:35,194 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=21, ppid=20, state=SUCCESS; OpenRegionProcedure 30c77692ccc4e0df458e1f12cc34f3f8, server=jenkins-hbase3.apache.org,38147,1690029314593 in 179 msec 2023-07-22T12:35:35,196 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=20, resume processing ppid=19 2023-07-22T12:35:35,196 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=20, ppid=19, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16900293305672, region=30c77692ccc4e0df458e1f12cc34f3f8, ASSIGN in 339 msec 2023-07-22T12:35:35,197 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:test-16900293305672 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:35:35,197 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16900293305672","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029335197"}]},"ts":"1690029335197"} 2023-07-22T12:35:35,199 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16900293305672, state=ENABLED in hbase:meta 2023-07-22T12:35:35,201 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16900293305672 execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:35:35,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=19, state=SUCCESS; CreateTableProcedure table=ns3:test-16900293305672 in 1.1940 sec 2023-07-22T12:35:36,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-07-22T12:35:36,121 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: ns3:test-16900293305672, procId: 19 completed 2023-07-22T12:35:36,121 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns3:test-16900293305672 get assigned. Timeout = 60000ms 2023-07-22T12:35:36,124 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-07-22T12:35:36,128 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.HBaseTestingUtility(3581): All regions for table ns3:test-16900293305672 assigned to meta. Checking AM states. 2023-07-22T12:35:36,128 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-07-22T12:35:36,129 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.HBaseTestingUtility(3601): All regions for table ns3:test-16900293305672 assigned. 2023-07-22T12:35:36,131 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns4:test-16900293305673', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:35:36,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] procedure2.ProcedureExecutor(1033): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns4:test-16900293305673 2023-07-22T12:35:36,136 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16900293305673 execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:35:36,136 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:36,136 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns4" qualifier: "test-16900293305673" procId is: 22 2023-07-22T12:35:36,137 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns4:test-16900293305673 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:35:36,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-07-22T12:35:36,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-07-22T12:35:36,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-07-22T12:35:36,552 INFO [RegionOpenAndInit-ns4:test-16900293305673-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 55c1ed70a7e1f74fa5c816e79d65220e, NAME => 'ns4:test-16900293305673,,1690029336131.55c1ed70a7e1f74fa5c816e79d65220e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-16900293305673', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f 2023-07-22T12:35:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-07-22T12:35:36,969 DEBUG [RegionOpenAndInit-ns4:test-16900293305673-pool-0 {}] regionserver.HRegion(888): Instantiated ns4:test-16900293305673,,1690029336131.55c1ed70a7e1f74fa5c816e79d65220e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:35:36,969 DEBUG [RegionOpenAndInit-ns4:test-16900293305673-pool-0 {}] regionserver.HRegion(1668): Closing 55c1ed70a7e1f74fa5c816e79d65220e, disabling compactions & flushes 2023-07-22T12:35:36,969 INFO [RegionOpenAndInit-ns4:test-16900293305673-pool-0 {}] regionserver.HRegion(1690): Closing region ns4:test-16900293305673,,1690029336131.55c1ed70a7e1f74fa5c816e79d65220e. 2023-07-22T12:35:36,969 DEBUG [RegionOpenAndInit-ns4:test-16900293305673-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16900293305673,,1690029336131.55c1ed70a7e1f74fa5c816e79d65220e. 2023-07-22T12:35:36,969 DEBUG [RegionOpenAndInit-ns4:test-16900293305673-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16900293305673,,1690029336131.55c1ed70a7e1f74fa5c816e79d65220e. after waiting 0 ms 2023-07-22T12:35:36,969 DEBUG [RegionOpenAndInit-ns4:test-16900293305673-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16900293305673,,1690029336131.55c1ed70a7e1f74fa5c816e79d65220e. 2023-07-22T12:35:36,969 INFO [RegionOpenAndInit-ns4:test-16900293305673-pool-0 {}] regionserver.HRegion(1909): Closed ns4:test-16900293305673,,1690029336131.55c1ed70a7e1f74fa5c816e79d65220e. 2023-07-22T12:35:36,969 DEBUG [RegionOpenAndInit-ns4:test-16900293305673-pool-0 {}] regionserver.HRegion(1622): Region close journal for 55c1ed70a7e1f74fa5c816e79d65220e: 2023-07-22T12:35:36,971 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns4:test-16900293305673 execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:35:36,971 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns4:test-16900293305673,,1690029336131.55c1ed70a7e1f74fa5c816e79d65220e.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1690029336971"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029336971"}]},"ts":"1690029336971"} 2023-07-22T12:35:36,973 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:35:36,974 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns4:test-16900293305673 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:35:36,974 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16900293305673","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029336974"}]},"ts":"1690029336974"} 2023-07-22T12:35:36,975 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16900293305673, state=ENABLING in hbase:meta 2023-07-22T12:35:36,979 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16900293305673, region=55c1ed70a7e1f74fa5c816e79d65220e, ASSIGN}] 2023-07-22T12:35:36,982 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16900293305673, region=55c1ed70a7e1f74fa5c816e79d65220e, ASSIGN 2023-07-22T12:35:36,983 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns4:test-16900293305673, region=55c1ed70a7e1f74fa5c816e79d65220e, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38147,1690029314593; forceNewPlan=false, retain=false 2023-07-22T12:35:37,135 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=55c1ed70a7e1f74fa5c816e79d65220e, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:37,137 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE; OpenRegionProcedure 55c1ed70a7e1f74fa5c816e79d65220e, server=jenkins-hbase3.apache.org,38147,1690029314593}] 2023-07-22T12:35:37,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-07-22T12:35:37,295 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(130): Open ns4:test-16900293305673,,1690029336131.55c1ed70a7e1f74fa5c816e79d65220e. 2023-07-22T12:35:37,295 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7254): Opening region: {ENCODED => 55c1ed70a7e1f74fa5c816e79d65220e, NAME => 'ns4:test-16900293305673,,1690029336131.55c1ed70a7e1f74fa5c816e79d65220e.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:35:37,295 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:35:37,296 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16900293305673 55c1ed70a7e1f74fa5c816e79d65220e 2023-07-22T12:35:37,296 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(888): Instantiated ns4:test-16900293305673,,1690029336131.55c1ed70a7e1f74fa5c816e79d65220e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:35:37,296 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7296): checking encryption for 55c1ed70a7e1f74fa5c816e79d65220e 2023-07-22T12:35:37,296 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7299): checking classloading for 55c1ed70a7e1f74fa5c816e79d65220e 2023-07-22T12:35:37,298 INFO [StoreOpener-55c1ed70a7e1f74fa5c816e79d65220e-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 55c1ed70a7e1f74fa5c816e79d65220e 2023-07-22T12:35:37,299 INFO [StoreOpener-55c1ed70a7e1f74fa5c816e79d65220e-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 55c1ed70a7e1f74fa5c816e79d65220e columnFamilyName f 2023-07-22T12:35:37,299 DEBUG [StoreOpener-55c1ed70a7e1f74fa5c816e79d65220e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:37,300 INFO [StoreOpener-55c1ed70a7e1f74fa5c816e79d65220e-1 {}] regionserver.HStore(324): Store=55c1ed70a7e1f74fa5c816e79d65220e/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:35:37,301 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/ns4/test-16900293305673/55c1ed70a7e1f74fa5c816e79d65220e 2023-07-22T12:35:37,301 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/ns4/test-16900293305673/55c1ed70a7e1f74fa5c816e79d65220e 2023-07-22T12:35:37,307 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1076): writing seq id for 55c1ed70a7e1f74fa5c816e79d65220e 2023-07-22T12:35:37,310 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/ns4/test-16900293305673/55c1ed70a7e1f74fa5c816e79d65220e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:35:37,311 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): Opened 55c1ed70a7e1f74fa5c816e79d65220e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10237419680, jitterRate=-0.04656599462032318}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:35:37,311 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(992): Region open journal for 55c1ed70a7e1f74fa5c816e79d65220e: 2023-07-22T12:35:37,312 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2552): Post open deploy tasks for ns4:test-16900293305673,,1690029336131.55c1ed70a7e1f74fa5c816e79d65220e., pid=24, masterSystemTime=1690029337290 2023-07-22T12:35:37,314 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2579): Finished post open deploy task for ns4:test-16900293305673,,1690029336131.55c1ed70a7e1f74fa5c816e79d65220e. 2023-07-22T12:35:37,314 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(158): Opened ns4:test-16900293305673,,1690029336131.55c1ed70a7e1f74fa5c816e79d65220e. 2023-07-22T12:35:37,315 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=55c1ed70a7e1f74fa5c816e79d65220e, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:37,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=24, resume processing ppid=23 2023-07-22T12:35:37,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=24, ppid=23, state=SUCCESS; OpenRegionProcedure 55c1ed70a7e1f74fa5c816e79d65220e, server=jenkins-hbase3.apache.org,38147,1690029314593 in 179 msec 2023-07-22T12:35:37,321 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=23, resume processing ppid=22 2023-07-22T12:35:37,321 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=23, ppid=22, state=SUCCESS; TransitRegionStateProcedure table=ns4:test-16900293305673, region=55c1ed70a7e1f74fa5c816e79d65220e, ASSIGN in 340 msec 2023-07-22T12:35:37,322 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns4:test-16900293305673 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:35:37,322 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16900293305673","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029337322"}]},"ts":"1690029337322"} 2023-07-22T12:35:37,323 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16900293305673, state=ENABLED in hbase:meta 2023-07-22T12:35:37,325 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16900293305673 execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:35:37,327 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=22, state=SUCCESS; CreateTableProcedure table=ns4:test-16900293305673 in 1.1940 sec 2023-07-22T12:35:37,781 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:35:38,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-07-22T12:35:38,245 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: ns4:test-16900293305673, procId: 22 completed 2023-07-22T12:35:38,246 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns4:test-16900293305673 get assigned. Timeout = 60000ms 2023-07-22T12:35:38,246 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-07-22T12:35:38,249 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.HBaseTestingUtility(3581): All regions for table ns4:test-16900293305673 assigned to meta. Checking AM states. 2023-07-22T12:35:38,250 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-07-22T12:35:38,250 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.HBaseTestingUtility(3601): All regions for table ns4:test-16900293305673 assigned. 2023-07-22T12:35:38,250 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x41cc3954 to 127.0.0.1:54609 2023-07-22T12:35:38,250 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:35:38,310 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.ResourceChecker(147): before: backup.TestRemoteRestore#testFullRestoreRemote Thread=816, OpenFileDescriptor=878, MaxFileDescriptor=60000, SystemLoadAverage=104, ProcessCount=169, AvailableMemoryMB=16717 2023-07-22T12:35:38,310 WARN [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.ResourceChecker(130): Thread=816 is superior to 500 2023-07-22T12:35:38,312 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] backup.TestRemoteRestore(70): test remote full backup on a single table 2023-07-22T12:35:38,312 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7fc1f5ff to 127.0.0.1:54609 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:35:38,317 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1201826a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:35:38,339 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:35:38,341 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:43408, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:35:38,353 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:35:38,354 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:41856, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:35:38,364 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'backup'} 2023-07-22T12:35:38,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] procedure2.ProcedureExecutor(1033): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=backup 2023-07-22T12:35:38,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=25 2023-07-22T12:35:38,374 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:35:38,378 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=25, state=SUCCESS; CreateNamespaceProcedure, namespace=backup in 12 msec 2023-07-22T12:35:38,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=25 2023-07-22T12:35:38,484 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:35:38,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] procedure2.ProcedureExecutor(1033): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system 2023-07-22T12:35:38,490 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:35:38,490 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:38,490 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2023-07-22T12:35:38,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-07-22T12:35:38,492 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:35:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-07-22T12:35:38,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-07-22T12:35:38,912 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 5c9301698cb41dfb7fd9be1c9c6a1f19, NAME => 'backup:system,,1690029338484.5c9301698cb41dfb7fd9be1c9c6a1f19.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f 2023-07-22T12:35:38,929 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system,,1690029338484.5c9301698cb41dfb7fd9be1c9c6a1f19.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:35:38,929 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1668): Closing 5c9301698cb41dfb7fd9be1c9c6a1f19, disabling compactions & flushes 2023-07-22T12:35:38,930 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system,,1690029338484.5c9301698cb41dfb7fd9be1c9c6a1f19. 2023-07-22T12:35:38,930 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1690029338484.5c9301698cb41dfb7fd9be1c9c6a1f19. 2023-07-22T12:35:38,930 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1690029338484.5c9301698cb41dfb7fd9be1c9c6a1f19. after waiting 0 ms 2023-07-22T12:35:38,930 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1690029338484.5c9301698cb41dfb7fd9be1c9c6a1f19. 2023-07-22T12:35:38,930 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1909): Closed backup:system,,1690029338484.5c9301698cb41dfb7fd9be1c9c6a1f19. 2023-07-22T12:35:38,930 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1622): Region close journal for 5c9301698cb41dfb7fd9be1c9c6a1f19: 2023-07-22T12:35:38,931 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:35:38,932 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1690029338484.5c9301698cb41dfb7fd9be1c9c6a1f19.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1690029338932"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029338932"}]},"ts":"1690029338932"} 2023-07-22T12:35:38,934 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:35:38,935 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:35:38,935 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029338935"}]},"ts":"1690029338935"} 2023-07-22T12:35:38,937 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-07-22T12:35:38,940 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=5c9301698cb41dfb7fd9be1c9c6a1f19, ASSIGN}] 2023-07-22T12:35:38,943 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=5c9301698cb41dfb7fd9be1c9c6a1f19, ASSIGN 2023-07-22T12:35:38,944 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=5c9301698cb41dfb7fd9be1c9c6a1f19, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38147,1690029314593; forceNewPlan=false, retain=false 2023-07-22T12:35:39,095 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=5c9301698cb41dfb7fd9be1c9c6a1f19, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:39,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-07-22T12:35:39,097 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; OpenRegionProcedure 5c9301698cb41dfb7fd9be1c9c6a1f19, server=jenkins-hbase3.apache.org,38147,1690029314593}] 2023-07-22T12:35:39,253 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(130): Open backup:system,,1690029338484.5c9301698cb41dfb7fd9be1c9c6a1f19. 2023-07-22T12:35:39,254 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7254): Opening region: {ENCODED => 5c9301698cb41dfb7fd9be1c9c6a1f19, NAME => 'backup:system,,1690029338484.5c9301698cb41dfb7fd9be1c9c6a1f19.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:35:39,254 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:35:39,254 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 5c9301698cb41dfb7fd9be1c9c6a1f19 2023-07-22T12:35:39,254 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(888): Instantiated backup:system,,1690029338484.5c9301698cb41dfb7fd9be1c9c6a1f19.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:35:39,254 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7296): checking encryption for 5c9301698cb41dfb7fd9be1c9c6a1f19 2023-07-22T12:35:39,254 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7299): checking classloading for 5c9301698cb41dfb7fd9be1c9c6a1f19 2023-07-22T12:35:39,256 INFO [StoreOpener-5c9301698cb41dfb7fd9be1c9c6a1f19-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 5c9301698cb41dfb7fd9be1c9c6a1f19 2023-07-22T12:35:39,258 INFO [StoreOpener-5c9301698cb41dfb7fd9be1c9c6a1f19-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5c9301698cb41dfb7fd9be1c9c6a1f19 columnFamilyName meta 2023-07-22T12:35:39,258 DEBUG [StoreOpener-5c9301698cb41dfb7fd9be1c9c6a1f19-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:39,258 INFO [StoreOpener-5c9301698cb41dfb7fd9be1c9c6a1f19-1 {}] regionserver.HStore(324): Store=5c9301698cb41dfb7fd9be1c9c6a1f19/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:35:39,259 INFO [StoreOpener-5c9301698cb41dfb7fd9be1c9c6a1f19-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 5c9301698cb41dfb7fd9be1c9c6a1f19 2023-07-22T12:35:39,260 INFO [StoreOpener-5c9301698cb41dfb7fd9be1c9c6a1f19-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5c9301698cb41dfb7fd9be1c9c6a1f19 columnFamilyName session 2023-07-22T12:35:39,260 DEBUG [StoreOpener-5c9301698cb41dfb7fd9be1c9c6a1f19-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:39,261 INFO [StoreOpener-5c9301698cb41dfb7fd9be1c9c6a1f19-1 {}] regionserver.HStore(324): Store=5c9301698cb41dfb7fd9be1c9c6a1f19/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:35:39,262 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19 2023-07-22T12:35:39,262 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19 2023-07-22T12:35:39,263 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-07-22T12:35:39,265 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1076): writing seq id for 5c9301698cb41dfb7fd9be1c9c6a1f19 2023-07-22T12:35:39,267 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:35:39,267 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): Opened 5c9301698cb41dfb7fd9be1c9c6a1f19; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11272370880, jitterRate=0.04982134699821472}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-07-22T12:35:39,268 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(992): Region open journal for 5c9301698cb41dfb7fd9be1c9c6a1f19: 2023-07-22T12:35:39,269 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system,,1690029338484.5c9301698cb41dfb7fd9be1c9c6a1f19., pid=28, masterSystemTime=1690029339250 2023-07-22T12:35:39,271 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system,,1690029338484.5c9301698cb41dfb7fd9be1c9c6a1f19. 2023-07-22T12:35:39,271 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(158): Opened backup:system,,1690029338484.5c9301698cb41dfb7fd9be1c9c6a1f19. 2023-07-22T12:35:39,271 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=5c9301698cb41dfb7fd9be1c9c6a1f19, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:39,275 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=28, resume processing ppid=27 2023-07-22T12:35:39,275 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=28, ppid=27, state=SUCCESS; OpenRegionProcedure 5c9301698cb41dfb7fd9be1c9c6a1f19, server=jenkins-hbase3.apache.org,38147,1690029314593 in 176 msec 2023-07-22T12:35:39,277 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=27, resume processing ppid=26 2023-07-22T12:35:39,277 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=27, ppid=26, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=5c9301698cb41dfb7fd9be1c9c6a1f19, ASSIGN in 335 msec 2023-07-22T12:35:39,278 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:35:39,278 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029339278"}]},"ts":"1690029339278"} 2023-07-22T12:35:39,279 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-07-22T12:35:39,282 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:35:39,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=26, state=SUCCESS; CreateTableProcedure table=backup:system in 797 msec 2023-07-22T12:35:39,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-07-22T12:35:39,598 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: backup:system, procId: 26 completed 2023-07-22T12:35:39,612 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:35:39,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] procedure2.ProcedureExecutor(1033): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system_bulk 2023-07-22T12:35:39,616 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:35:39,616 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:39,616 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2023-07-22T12:35:39,617 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:35:39,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-07-22T12:35:39,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-07-22T12:35:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-07-22T12:35:40,029 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 678d09e574bb66f63f5cd63b46064d40, NAME => 'backup:system_bulk,,1690029339611.678d09e574bb66f63f5cd63b46064d40.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f 2023-07-22T12:35:40,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-07-22T12:35:40,441 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1690029339611.678d09e574bb66f63f5cd63b46064d40.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:35:40,441 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1668): Closing 678d09e574bb66f63f5cd63b46064d40, disabling compactions & flushes 2023-07-22T12:35:40,441 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1690029339611.678d09e574bb66f63f5cd63b46064d40. 2023-07-22T12:35:40,441 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1690029339611.678d09e574bb66f63f5cd63b46064d40. 2023-07-22T12:35:40,441 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1690029339611.678d09e574bb66f63f5cd63b46064d40. after waiting 0 ms 2023-07-22T12:35:40,442 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1690029339611.678d09e574bb66f63f5cd63b46064d40. 2023-07-22T12:35:40,442 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1909): Closed backup:system_bulk,,1690029339611.678d09e574bb66f63f5cd63b46064d40. 2023-07-22T12:35:40,442 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1622): Region close journal for 678d09e574bb66f63f5cd63b46064d40: 2023-07-22T12:35:40,443 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:35:40,443 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system_bulk,,1690029339611.678d09e574bb66f63f5cd63b46064d40.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1690029340443"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029340443"}]},"ts":"1690029340443"} 2023-07-22T12:35:40,445 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:35:40,446 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:35:40,447 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029340447"}]},"ts":"1690029340447"} 2023-07-22T12:35:40,448 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2023-07-22T12:35:40,452 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=678d09e574bb66f63f5cd63b46064d40, ASSIGN}] 2023-07-22T12:35:40,455 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=678d09e574bb66f63f5cd63b46064d40, ASSIGN 2023-07-22T12:35:40,456 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system_bulk, region=678d09e574bb66f63f5cd63b46064d40, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38147,1690029314593; forceNewPlan=false, retain=false 2023-07-22T12:35:40,538 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-16900293305672' 2023-07-22T12:35:40,539 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-16900293305673' 2023-07-22T12:35:40,539 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-07-22T12:35:40,608 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=678d09e574bb66f63f5cd63b46064d40, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:40,610 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE; OpenRegionProcedure 678d09e574bb66f63f5cd63b46064d40, server=jenkins-hbase3.apache.org,38147,1690029314593}] 2023-07-22T12:35:40,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-07-22T12:35:40,766 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(130): Open backup:system_bulk,,1690029339611.678d09e574bb66f63f5cd63b46064d40. 2023-07-22T12:35:40,766 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7254): Opening region: {ENCODED => 678d09e574bb66f63f5cd63b46064d40, NAME => 'backup:system_bulk,,1690029339611.678d09e574bb66f63f5cd63b46064d40.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:35:40,766 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:35:40,766 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk 678d09e574bb66f63f5cd63b46064d40 2023-07-22T12:35:40,766 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1690029339611.678d09e574bb66f63f5cd63b46064d40.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:35:40,767 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7296): checking encryption for 678d09e574bb66f63f5cd63b46064d40 2023-07-22T12:35:40,767 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7299): checking classloading for 678d09e574bb66f63f5cd63b46064d40 2023-07-22T12:35:40,768 INFO [StoreOpener-678d09e574bb66f63f5cd63b46064d40-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 678d09e574bb66f63f5cd63b46064d40 2023-07-22T12:35:40,770 INFO [StoreOpener-678d09e574bb66f63f5cd63b46064d40-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 678d09e574bb66f63f5cd63b46064d40 columnFamilyName meta 2023-07-22T12:35:40,770 DEBUG [StoreOpener-678d09e574bb66f63f5cd63b46064d40-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:40,770 INFO [StoreOpener-678d09e574bb66f63f5cd63b46064d40-1 {}] regionserver.HStore(324): Store=678d09e574bb66f63f5cd63b46064d40/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:35:40,771 INFO [StoreOpener-678d09e574bb66f63f5cd63b46064d40-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 678d09e574bb66f63f5cd63b46064d40 2023-07-22T12:35:40,772 INFO [StoreOpener-678d09e574bb66f63f5cd63b46064d40-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 678d09e574bb66f63f5cd63b46064d40 columnFamilyName session 2023-07-22T12:35:40,772 DEBUG [StoreOpener-678d09e574bb66f63f5cd63b46064d40-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:40,773 INFO [StoreOpener-678d09e574bb66f63f5cd63b46064d40-1 {}] regionserver.HStore(324): Store=678d09e574bb66f63f5cd63b46064d40/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:35:40,774 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system_bulk/678d09e574bb66f63f5cd63b46064d40 2023-07-22T12:35:40,774 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system_bulk/678d09e574bb66f63f5cd63b46064d40 2023-07-22T12:35:40,776 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-07-22T12:35:40,777 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1076): writing seq id for 678d09e574bb66f63f5cd63b46064d40 2023-07-22T12:35:40,780 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system_bulk/678d09e574bb66f63f5cd63b46064d40/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:35:40,781 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): Opened 678d09e574bb66f63f5cd63b46064d40; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11820825920, jitterRate=0.10090020298957825}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-07-22T12:35:40,782 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(992): Region open journal for 678d09e574bb66f63f5cd63b46064d40: 2023-07-22T12:35:40,783 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system_bulk,,1690029339611.678d09e574bb66f63f5cd63b46064d40., pid=31, masterSystemTime=1690029340762 2023-07-22T12:35:40,784 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system_bulk,,1690029339611.678d09e574bb66f63f5cd63b46064d40. 2023-07-22T12:35:40,784 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(158): Opened backup:system_bulk,,1690029339611.678d09e574bb66f63f5cd63b46064d40. 2023-07-22T12:35:40,785 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=678d09e574bb66f63f5cd63b46064d40, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:40,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=31, resume processing ppid=30 2023-07-22T12:35:40,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=31, ppid=30, state=SUCCESS; OpenRegionProcedure 678d09e574bb66f63f5cd63b46064d40, server=jenkins-hbase3.apache.org,38147,1690029314593 in 176 msec 2023-07-22T12:35:40,790 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=30, resume processing ppid=29 2023-07-22T12:35:40,790 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=30, ppid=29, state=SUCCESS; TransitRegionStateProcedure table=backup:system_bulk, region=678d09e574bb66f63f5cd63b46064d40, ASSIGN in 336 msec 2023-07-22T12:35:40,791 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:35:40,791 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029340791"}]},"ts":"1690029340791"} 2023-07-22T12:35:40,793 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2023-07-22T12:35:40,795 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:35:40,797 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=29, state=SUCCESS; CreateTableProcedure table=backup:system_bulk in 1.1830 sec 2023-07-22T12:35:41,691 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-07-22T12:35:41,692 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-07-22T12:35:41,692 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16900293305673 2023-07-22T12:35:41,692 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-16900293305673 Metrics about Tables on a single HBase RegionServer 2023-07-22T12:35:41,693 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-07-22T12:35:41,693 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2023-07-22T12:35:41,693 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16900293305672 2023-07-22T12:35:41,694 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-16900293305672 Metrics about Tables on a single HBase RegionServer 2023-07-22T12:35:41,694 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16900293305671 2023-07-22T12:35:41,694 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-16900293305671 Metrics about Tables on a single HBase RegionServer 2023-07-22T12:35:41,695 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1690029330567 2023-07-22T12:35:41,695 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1690029330567 Metrics about Tables on a single HBase RegionServer 2023-07-22T12:35:41,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-07-22T12:35:41,724 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: backup:system_bulk, procId: 29 completed 2023-07-22T12:35:41,737 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:35:41,784 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:35:41,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029341784 (current time:1690029341784). 2023-07-22T12:35:41,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:35:41,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:35:41,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x594cb113 to 127.0.0.1:54609 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:35:41,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6555bf2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:35:41,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:35:41,795 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:49708, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:35:41,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x594cb113 to 127.0.0.1:54609 2023-07-22T12:35:41,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:35:41,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:35:41,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] procedure2.ProcedureExecutor(1033): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:35:41,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-07-22T12:35:41,812 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:35:41,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:35:41,817 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:35:41,820 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:35:41,828 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:35:41,831 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 5c9301698cb41dfb7fd9be1c9c6a1f19}] 2023-07-22T12:35:41,837 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 5c9301698cb41dfb7fd9be1c9c6a1f19 2023-07-22T12:35:41,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:35:41,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38147 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2023-07-22T12:35:41,997 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690029338484.5c9301698cb41dfb7fd9be1c9c6a1f19. 2023-07-22T12:35:42,000 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2819): Flushing 5c9301698cb41dfb7fd9be1c9c6a1f19 2/2 column families, dataSize=45 B heapSize=632 B 2023-07-22T12:35:42,075 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/.tmp/session/d3f88af37ddf42418c7fa151a0dd871a is 49, key is activesession:/session:c/1690029341769/Put/seqid=0 2023-07-22T12:35:42,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:35:42,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:35:42,496 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/.tmp/session/d3f88af37ddf42418c7fa151a0dd871a 2023-07-22T12:35:42,560 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: d3f88af37ddf42418c7fa151a0dd871a 2023-07-22T12:35:42,564 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/.tmp/session/d3f88af37ddf42418c7fa151a0dd871a as hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/session/d3f88af37ddf42418c7fa151a0dd871a 2023-07-22T12:35:42,571 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/session/d3f88af37ddf42418c7fa151a0dd871a, entries=1, sequenceid=5, filesize=4.9 K 2023-07-22T12:35:42,575 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3022): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for 5c9301698cb41dfb7fd9be1c9c6a1f19 in 576ms, sequenceid=5, compaction requested=false 2023-07-22T12:35:42,578 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2520): Flush status journal for 5c9301698cb41dfb7fd9be1c9c6a1f19: 2023-07-22T12:35:42,578 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690029338484.5c9301698cb41dfb7fd9be1c9c6a1f19. for snapshot_backup_system completed. 2023-07-22T12:35:42,579 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690029338484.5c9301698cb41dfb7fd9be1c9c6a1f19.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:35:42,585 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:35:42,590 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-07-22T12:35:42,590 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/session/d3f88af37ddf42418c7fa151a0dd871a] hfiles 2023-07-22T12:35:42,590 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/session/d3f88af37ddf42418c7fa151a0dd871a for snapshot=snapshot_backup_system 2023-07-22T12:35:42,798 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:35:42,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:35:43,005 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690029338484.5c9301698cb41dfb7fd9be1c9c6a1f19. 2023-07-22T12:35:43,008 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=33 2023-07-22T12:35:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.HMaster(3994): Remote procedure done, pid=33 2023-07-22T12:35:43,012 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 5c9301698cb41dfb7fd9be1c9c6a1f19 2023-07-22T12:35:43,014 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 5c9301698cb41dfb7fd9be1c9c6a1f19 2023-07-22T12:35:43,017 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=33, resume processing ppid=32 2023-07-22T12:35:43,017 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=33, ppid=32, state=SUCCESS; SnapshotRegionProcedure 5c9301698cb41dfb7fd9be1c9c6a1f19 in 1.1840 sec 2023-07-22T12:35:43,017 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:35:43,019 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:35:43,022 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:35:43,022 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:35:43,025 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:35:43,452 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:35:43,465 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:35:43,465 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:35:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:35:44,294 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:35:44,294 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-07-22T12:35:44,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=32, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.4890 sec 2023-07-22T12:35:45,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:35:45,920 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: backup:system, procId: 32 completed 2023-07-22T12:35:45,920 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] impl.TableBackupClient(115): Backup backup_1690029338321 started at 1690029345920. 2023-07-22T12:35:45,942 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.TableBackupClient(119): Backup session backup_1690029338321 has been started. 2023-07-22T12:35:45,944 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-07-22T12:35:45,955 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-07-22T12:35:45,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-07-22T12:35:45,963 INFO [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-07-22T12:35:45,963 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-07-22T12:35:45,964 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-07-22T12:35:45,964 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-07-22T12:35:45,965 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-07-22T12:35:45,965 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:35:45,966 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:45,966 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-07-22T12:35:45,966 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:35:45,966 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:35:45,966 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:45,967 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-07-22T12:35:45,967 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-07-22T12:35:45,967 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-07-22T12:35:45,967 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-07-22T12:35:45,967 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:35:45,967 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-07-22T12:35:45,969 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-07-22T12:35:45,969 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-07-22T12:35:45,970 DEBUG [member: 'jenkins-hbase3.apache.org,38147,1690029314593' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-07-22T12:35:45,970 DEBUG [member: 'jenkins-hbase3.apache.org,38147,1690029314593' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-07-22T12:35:45,970 DEBUG [member: 'jenkins-hbase3.apache.org,38147,1690029314593' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-07-22T12:35:45,970 DEBUG [member: 'jenkins-hbase3.apache.org,38147,1690029314593' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-07-22T12:35:45,970 DEBUG [member: 'jenkins-hbase3.apache.org,38147,1690029314593' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,38147,1690029314593' joining acquired barrier for procedure (rolllog) in zk 2023-07-22T12:35:45,972 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:45,972 DEBUG [member: 'jenkins-hbase3.apache.org,38147,1690029314593' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-07-22T12:35:45,972 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:45,972 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:35:45,972 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:35:45,972 DEBUG [member: 'jenkins-hbase3.apache.org,38147,1690029314593' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-07-22T12:35:45,972 DEBUG [member: 'jenkins-hbase3.apache.org,38147,1690029314593' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-07-22T12:35:45,972 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:35:45,973 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:35:45,973 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:35:45,973 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:45,974 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:35:45,974 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,38147,1690029314593' joining acquired barrier for procedure 'rolllog' on coordinator 2023-07-22T12:35:45,974 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@45f9e78a[Count = 0] remaining members to acquire global barrier 2023-07-22T12:35:45,974 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-07-22T12:35:45,974 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-07-22T12:35:45,975 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:35:45,975 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-07-22T12:35:45,975 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-07-22T12:35:45,976 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:45,976 DEBUG [member: 'jenkins-hbase3.apache.org,38147,1690029314593' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-07-22T12:35:45,976 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-07-22T12:35:45,976 DEBUG [member: 'jenkins-hbase3.apache.org,38147,1690029314593' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-07-22T12:35:45,976 DEBUG [rs(jenkins-hbase3.apache.org,38147,1690029314593)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:45,977 INFO [rs(jenkins-hbase3.apache.org,38147,1690029314593)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1690029316733 highest: 1690029316733 on jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:45,977 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C38147%2C1690029314593.meta:.meta(num 1690029317777) roll requested 2023-07-22T12:35:45,988 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/WALs/jenkins-hbase3.apache.org,38147,1690029314593/jenkins-hbase3.apache.org%2C38147%2C1690029314593.meta.1690029345977.meta, exclude list is [], retry=0 2023-07-22T12:35:45,995 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38553,DS-c76e90c8-39fa-4f11-ba17-53eb00da3429,DISK] 2023-07-22T12:35:45,998 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/WALs/jenkins-hbase3.apache.org,38147,1690029314593/jenkins-hbase3.apache.org%2C38147%2C1690029314593.meta.1690029317777.meta with entries=36, filesize=11.13 KB; new WAL /user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/WALs/jenkins-hbase3.apache.org,38147,1690029314593/jenkins-hbase3.apache.org%2C38147%2C1690029314593.meta.1690029345977.meta 2023-07-22T12:35:45,999 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38553,DS-c76e90c8-39fa-4f11-ba17-53eb00da3429,DISK]] 2023-07-22T12:35:45,999 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/WALs/jenkins-hbase3.apache.org,38147,1690029314593/jenkins-hbase3.apache.org%2C38147%2C1690029314593.meta.1690029317777.meta is not closed yet, will try archiving it next time 2023-07-22T12:35:45,999 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C38147%2C1690029314593:(num 1690029316733) roll requested 2023-07-22T12:35:46,013 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/WALs/jenkins-hbase3.apache.org,38147,1690029314593/jenkins-hbase3.apache.org%2C38147%2C1690029314593.1690029346000, exclude list is [], retry=0 2023-07-22T12:35:46,017 DEBUG [rs(jenkins-hbase3.apache.org,38147,1690029314593)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-07-22T12:35:46,017 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38553,DS-c76e90c8-39fa-4f11-ba17-53eb00da3429,DISK] 2023-07-22T12:35:46,017 INFO [rs(jenkins-hbase3.apache.org,38147,1690029314593)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1690029346000 on jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:46,018 DEBUG [rs(jenkins-hbase3.apache.org,38147,1690029314593)-backup-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:35:46,020 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:42932, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-07-22T12:35:46,022 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/WALs/jenkins-hbase3.apache.org,38147,1690029314593/jenkins-hbase3.apache.org%2C38147%2C1690029314593.1690029316733 with entries=19, filesize=3.99 KB; new WAL /user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/WALs/jenkins-hbase3.apache.org,38147,1690029314593/jenkins-hbase3.apache.org%2C38147%2C1690029314593.1690029346000 2023-07-22T12:35:46,022 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38553,DS-c76e90c8-39fa-4f11-ba17-53eb00da3429,DISK]] 2023-07-22T12:35:46,022 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/WALs/jenkins-hbase3.apache.org,38147,1690029314593/jenkins-hbase3.apache.org%2C38147%2C1690029314593.1690029316733 is not closed yet, will try archiving it next time 2023-07-22T12:35:46,042 DEBUG [member: 'jenkins-hbase3.apache.org,38147,1690029314593' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-07-22T12:35:46,042 DEBUG [member: 'jenkins-hbase3.apache.org,38147,1690029314593' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,38147,1690029314593' in zk 2023-07-22T12:35:46,045 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:46,045 DEBUG [member: 'jenkins-hbase3.apache.org,38147,1690029314593' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-07-22T12:35:46,045 DEBUG [member: 'jenkins-hbase3.apache.org,38147,1690029314593' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:35:46,045 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:46,045 DEBUG [member: 'jenkins-hbase3.apache.org,38147,1690029314593' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-07-22T12:35:46,046 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:35:46,046 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:35:46,047 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:35:46,047 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:35:46,047 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:35:46,048 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:46,048 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:35:46,048 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:35:46,048 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:46,049 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,38147,1690029314593': 2023-07-22T12:35:46,049 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,38147,1690029314593' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-07-22T12:35:46,049 INFO [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-07-22T12:35:46,049 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-07-22T12:35:46,049 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-07-22T12:35:46,049 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-07-22T12:35:46,049 INFO [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-07-22T12:35:46,051 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:35:46,051 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:35:46,051 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-07-22T12:35:46,051 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-07-22T12:35:46,051 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-07-22T12:35:46,051 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:35:46,051 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:35:46,051 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:35:46,051 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:35:46,051 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:35:46,051 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:46,051 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:35:46,052 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:35:46,052 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:35:46,052 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:35:46,053 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:35:46,053 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:46,053 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:46,053 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:35:46,054 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:35:46,054 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:46,059 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:46,060 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-07-22T12:35:46,060 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-07-22T12:35:46,060 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-07-22T12:35:46,060 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-07-22T12:35:46,060 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:35:46,060 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:46,060 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:35:46,060 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:35:46,060 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:35:46,060 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:35:46,060 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:35:46,060 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-07-22T12:35:46,061 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-07-22T12:35:46,061 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:35:46,061 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:35:46,063 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] client.HBaseAdmin(2751): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-07-22T12:35:46,064 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] client.HBaseAdmin(2759): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-07-22T12:35:46,164 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] client.HBaseAdmin(2765): Getting current status of procedure from master... 2023-07-22T12:35:46,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-07-22T12:35:46,173 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1690029346172_default_test-1690029330567 table=test-1690029330567 type=FLUSH ttl=0 } 2023-07-22T12:35:46,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029346173 (current time:1690029346173). 2023-07-22T12:35:46,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:35:46,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1690029346172_default_test-1690029330567 VERSION not specified, setting to 2 2023-07-22T12:35:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x37fb0e5a to 127.0.0.1:54609 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:35:46,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ec1f72c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:35:46,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:35:46,183 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:49722, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:35:46,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x37fb0e5a to 127.0.0.1:54609 2023-07-22T12:35:46,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:35:46,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:35:46,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] procedure2.ProcedureExecutor(1033): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029346172_default_test-1690029330567 table=test-1690029330567 type=FLUSH ttl=0 } 2023-07-22T12:35:46,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1690029346172_default_test-1690029330567 table=test-1690029330567 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-07-22T12:35:46,187 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029346172_default_test-1690029330567 table=test-1690029330567 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:35:46,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:35:46,188 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029346172_default_test-1690029330567 table=test-1690029330567 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:35:46,190 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029346172_default_test-1690029330567 table=test-1690029330567 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:35:46,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:35:46,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:35:46,540 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2023-07-22T12:35:46,602 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029346172_default_test-1690029330567 table=test-1690029330567 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:35:46,603 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure e2f9453920ffaa2bfd87a3743468e74e}] 2023-07-22T12:35:46,605 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure e2f9453920ffaa2bfd87a3743468e74e 2023-07-22T12:35:46,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38147 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2023-07-22T12:35:46,757 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1690029330567,,1690029331075.e2f9453920ffaa2bfd87a3743468e74e. 2023-07-22T12:35:46,759 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2819): Flushing e2f9453920ffaa2bfd87a3743468e74e 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-07-22T12:35:46,777 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/test-1690029330567/e2f9453920ffaa2bfd87a3743468e74e/.tmp/f/a663b1eb7c49421a9a8af7e19a9900bf is 37, key is row10/f:q1/1690029333239/Put/seqid=0 2023-07-22T12:35:46,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:35:47,183 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/test-1690029330567/e2f9453920ffaa2bfd87a3743468e74e/.tmp/f/a663b1eb7c49421a9a8af7e19a9900bf 2023-07-22T12:35:47,191 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a663b1eb7c49421a9a8af7e19a9900bf 2023-07-22T12:35:47,192 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/test-1690029330567/e2f9453920ffaa2bfd87a3743468e74e/.tmp/f/a663b1eb7c49421a9a8af7e19a9900bf as hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/test-1690029330567/e2f9453920ffaa2bfd87a3743468e74e/f/a663b1eb7c49421a9a8af7e19a9900bf 2023-07-22T12:35:47,199 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/test-1690029330567/e2f9453920ffaa2bfd87a3743468e74e/f/a663b1eb7c49421a9a8af7e19a9900bf, entries=99, sequenceid=103, filesize=8.2 K 2023-07-22T12:35:47,202 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for e2f9453920ffaa2bfd87a3743468e74e in 443ms, sequenceid=103, compaction requested=false 2023-07-22T12:35:47,203 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2520): Flush status journal for e2f9453920ffaa2bfd87a3743468e74e: 2023-07-22T12:35:47,203 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1690029330567,,1690029331075.e2f9453920ffaa2bfd87a3743468e74e. for snapshot_1690029346172_default_test-1690029330567 completed. 2023-07-22T12:35:47,203 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(240): Storing 'test-1690029330567,,1690029331075.e2f9453920ffaa2bfd87a3743468e74e.' region-info for snapshot=snapshot_1690029346172_default_test-1690029330567 2023-07-22T12:35:47,203 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:35:47,203 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/test-1690029330567/e2f9453920ffaa2bfd87a3743468e74e/f/a663b1eb7c49421a9a8af7e19a9900bf] hfiles 2023-07-22T12:35:47,203 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/test-1690029330567/e2f9453920ffaa2bfd87a3743468e74e/f/a663b1eb7c49421a9a8af7e19a9900bf for snapshot=snapshot_1690029346172_default_test-1690029330567 2023-07-22T12:35:47,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:35:47,609 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1690029330567,,1690029331075.e2f9453920ffaa2bfd87a3743468e74e. 2023-07-22T12:35:47,609 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=35 2023-07-22T12:35:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.HMaster(3994): Remote procedure done, pid=35 2023-07-22T12:35:47,610 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1690029346172_default_test-1690029330567 on region e2f9453920ffaa2bfd87a3743468e74e 2023-07-22T12:35:47,612 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure e2f9453920ffaa2bfd87a3743468e74e 2023-07-22T12:35:47,614 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=35, resume processing ppid=34 2023-07-22T12:35:47,614 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029346172_default_test-1690029330567 table=test-1690029330567 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:35:47,614 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=35, ppid=34, state=SUCCESS; SnapshotRegionProcedure e2f9453920ffaa2bfd87a3743468e74e in 1.0100 sec 2023-07-22T12:35:47,615 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029346172_default_test-1690029330567 table=test-1690029330567 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:35:47,616 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029346172_default_test-1690029330567 table=test-1690029330567 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:35:47,616 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1690029346172_default_test-1690029330567 2023-07-22T12:35:47,617 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/.hbase-snapshot/.tmp/snapshot_1690029346172_default_test-1690029330567 2023-07-22T12:35:48,028 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029346172_default_test-1690029330567 table=test-1690029330567 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:35:48,035 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029346172_default_test-1690029330567 table=test-1690029330567 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:35:48,036 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/.hbase-snapshot/.tmp/snapshot_1690029346172_default_test-1690029330567 to hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/.hbase-snapshot/snapshot_1690029346172_default_test-1690029330567 2023-07-22T12:35:48,247 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-07-22T12:35:48,249 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:42936, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-07-22T12:35:48,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:35:48,856 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029346172_default_test-1690029330567 table=test-1690029330567 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:35:48,856 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1690029346172_default_test-1690029330567 table=test-1690029330567 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-07-22T12:35:48,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=34, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029346172_default_test-1690029330567 table=test-1690029330567 type=FLUSH ttl=0 } in 2.6720 sec 2023-07-22T12:35:50,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:35:50,294 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: default:test-1690029330567, procId: 34 completed 2023-07-22T12:35:50,294 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1690029338321 2023-07-22T12:35:50,295 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-07-22T12:35:50,296 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-07-22T12:35:50,299 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1690029338321_test-1690029330567 2023-07-22T12:35:50,299 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1690029346172_default_test-1690029330567 to hdfs://localhost:38161/backupUT/backup_1690029338321/default/test-1690029330567/ 2023-07-22T12:35:50,309 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-07-22T12:35:50,339 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:38685, inputRoot=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f 2023-07-22T12:35:50,340 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,46137,1690029311532_-692059418_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:38161/backupUT/backup_1690029338321/default/test-1690029330567, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:38161/backupUT/backup_1690029338321/default/test-1690029330567/.hbase-snapshot/.tmp/snapshot_1690029346172_default_test-1690029330567 2023-07-22T12:35:50,340 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:38685, inputRoot=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f, snapshotDir=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/.hbase-snapshot/snapshot_1690029346172_default_test-1690029330567. 2023-07-22T12:35:50,352 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/.hbase-snapshot/snapshot_1690029346172_default_test-1690029330567 to hdfs://localhost:38161/backupUT/backup_1690029338321/default/test-1690029330567/.hbase-snapshot/.tmp/snapshot_1690029346172_default_test-1690029330567 2023-07-22T12:35:50,792 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:50,792 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:50,793 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:50,793 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:51,765 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/hadoop-7912308431464690426.jar 2023-07-22T12:35:51,765 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:51,765 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:51,766 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:51,766 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:51,767 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:51,767 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:51,768 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:51,768 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:51,768 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:51,769 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:35:51,770 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:35:51,770 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:35:51,771 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:35:51,771 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:35:51,772 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:35:51,772 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:35:51,772 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:35:51,773 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:35:51,773 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:35:51,774 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:35:51,774 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:35:51,778 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:35:51,779 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:35:51,779 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:35:51,779 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:35:51,780 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:35:51,780 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:35:51,781 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:35:51,847 WARN [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-07-22T12:35:51,872 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1690029346172_default_test-1690029330567' hfile list 2023-07-22T12:35:51,882 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-07-22T12:35:52,100 WARN [Listener at jenkins-hbase3.apache.org/44413 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/mapreduce.cluster.local.dir/1690029352048/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:35:52,100 WARN [Listener at jenkins-hbase3.apache.org/44413 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/mapreduce.cluster.local.dir/1690029352048/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:35:52,270 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-07-22T12:35:52,299 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/test-1690029330567/e2f9453920ffaa2bfd87a3743468e74e/f/a663b1eb7c49421a9a8af7e19a9900bf output=hdfs://localhost:38161/backupUT/backup_1690029338321/default/test-1690029330567/archive/data/default/test-1690029330567/e2f9453920ffaa2bfd87a3743468e74e/f/a663b1eb7c49421a9a8af7e19a9900bf 2023-07-22T12:35:52,299 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 8.007M/sec 2023-07-22T12:35:52,949 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-07-22T12:35:52,952 INFO [RS-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:39714, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2023-07-22T12:35:53,181 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-07-22T12:35:53,182 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-07-22T12:35:53,190 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1690029346172_default_test-1690029330567 2023-07-22T12:35:53,190 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1690029346172_default_test-1690029330567 finished. 2023-07-22T12:35:53,191 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.BackupSystemTable(1014): test-1690029330567 2023-07-22T12:35:53,262 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029338321 2023-07-22T12:35:53,262 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-07-22T12:35:53,685 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:38161/backupUT/backup_1690029338321/.backup.manifest 2023-07-22T12:35:53,685 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.TableBackupClient(358): Backup backup_1690029338321 finished: type=FULL,tablelist=test-1690029330567,targetRootDir=hdfs://localhost:38161/backupUT,startts=1690029345920,completets=1690029353259,bytescopied=0 2023-07-22T12:35:53,685 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-07-22T12:35:53,685 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1690029346172_default_test-1690029330567 2023-07-22T12:35:53,696 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1690029346172_default_test-1690029330567" 2023-07-22T12:35:53,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1690029346172_default_test-1690029330567 2023-07-22T12:35:53,703 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1690029346172_default_test-1690029330567 for backup backup_1690029338321 succeeded. 2023-07-22T12:35:53,704 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:35:53,720 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:35:53,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:35:53,729 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:35:53,738 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:35:53,762 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] impl.TableBackupClient(376): Backup backup_1690029338321 completed. 2023-07-22T12:35:53,762 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] client.ConnectionImplementation(2095): Closing master protocol: MasterService 2023-07-22T12:35:53,763 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7fc1f5ff to 127.0.0.1:54609 2023-07-22T12:35:53,763 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:35:53,763 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] backup.TestRemoteRestore(73): backup complete 2023-07-22T12:35:53,768 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] impl.RestoreTablesClient(101): HBase table table1 does not exist. It will be created during restore process 2023-07-22T12:35:53,769 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38161/backupUT/backup_1690029338321/.backup.manifest 2023-07-22T12:35:53,772 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029338321/.backup.manifest 2023-07-22T12:35:53,781 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38161/backupUT/backup_1690029338321/.backup.manifest 2023-07-22T12:35:53,783 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029338321/.backup.manifest 2023-07-22T12:35:53,783 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] impl.RestoreTablesClient(148): Restoring 'test-1690029330567' to 'table1' from full backup image hdfs://localhost:38161/backupUT/backup_1690029338321/default/test-1690029330567 2023-07-22T12:35:53,794 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] util.RestoreTool(487): Creating target table 'table1' 2023-07-22T12:35:53,794 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:38161/backupUT/backup_1690029338321/default/test-1690029330567/archive/data/default/test-1690029330567/e2f9453920ffaa2bfd87a3743468e74e 2023-07-22T12:35:53,795 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:38161/backupUT/backup_1690029338321/default/test-1690029330567/archive/data/default/test-1690029330567/e2f9453920ffaa2bfd87a3743468e74e/f in region [hdfs://localhost:38161/backupUT/backup_1690029338321/default/test-1690029330567/archive/data/default/test-1690029330567/e2f9453920ffaa2bfd87a3743468e74e] 2023-07-22T12:35:53,801 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] util.RestoreTool(442): Trying to figure out region boundaries hfile=hdfs://localhost:38161/backupUT/backup_1690029338321/default/test-1690029330567/archive/data/default/test-1690029330567/e2f9453920ffaa2bfd87a3743468e74e/f/a663b1eb7c49421a9a8af7e19a9900bf first=row0 last=row98 2023-07-22T12:35:53,801 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a663b1eb7c49421a9a8af7e19a9900bf 2023-07-22T12:35:53,811 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:35:53,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] procedure2.ProcedureExecutor(1033): Stored pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=table1 2023-07-22T12:35:53,815 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:35:53,816 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 36 2023-07-22T12:35:53,816 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:35:53,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-07-22T12:35:53,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-07-22T12:35:54,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-07-22T12:35:54,232 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => da5ae8ac17989cb406336333d6113a9e, NAME => 'table1,,1690029353811.da5ae8ac17989cb406336333d6113a9e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f 2023-07-22T12:35:54,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-07-22T12:35:54,640 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(888): Instantiated table1,,1690029353811.da5ae8ac17989cb406336333d6113a9e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:35:54,640 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1668): Closing da5ae8ac17989cb406336333d6113a9e, disabling compactions & flushes 2023-07-22T12:35:54,640 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1690): Closing region table1,,1690029353811.da5ae8ac17989cb406336333d6113a9e. 2023-07-22T12:35:54,640 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1690029353811.da5ae8ac17989cb406336333d6113a9e. 2023-07-22T12:35:54,640 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on table1,,1690029353811.da5ae8ac17989cb406336333d6113a9e. after waiting 0 ms 2023-07-22T12:35:54,640 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region table1,,1690029353811.da5ae8ac17989cb406336333d6113a9e. 2023-07-22T12:35:54,640 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1909): Closed table1,,1690029353811.da5ae8ac17989cb406336333d6113a9e. 2023-07-22T12:35:54,640 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1622): Region close journal for da5ae8ac17989cb406336333d6113a9e: 2023-07-22T12:35:54,642 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:35:54,642 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"table1,,1690029353811.da5ae8ac17989cb406336333d6113a9e.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1690029354642"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029354642"}]},"ts":"1690029354642"} 2023-07-22T12:35:54,645 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:35:54,646 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:35:54,646 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029354646"}]},"ts":"1690029354646"} 2023-07-22T12:35:54,648 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLING in hbase:meta 2023-07-22T12:35:54,655 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=da5ae8ac17989cb406336333d6113a9e, ASSIGN}] 2023-07-22T12:35:54,658 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=da5ae8ac17989cb406336333d6113a9e, ASSIGN 2023-07-22T12:35:54,659 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=table1, region=da5ae8ac17989cb406336333d6113a9e, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38147,1690029314593; forceNewPlan=false, retain=false 2023-07-22T12:35:54,811 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=da5ae8ac17989cb406336333d6113a9e, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:54,813 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE; OpenRegionProcedure da5ae8ac17989cb406336333d6113a9e, server=jenkins-hbase3.apache.org,38147,1690029314593}] 2023-07-22T12:35:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-07-22T12:35:54,969 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(130): Open table1,,1690029353811.da5ae8ac17989cb406336333d6113a9e. 2023-07-22T12:35:54,969 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7254): Opening region: {ENCODED => da5ae8ac17989cb406336333d6113a9e, NAME => 'table1,,1690029353811.da5ae8ac17989cb406336333d6113a9e.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:35:54,969 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:35:54,969 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 da5ae8ac17989cb406336333d6113a9e 2023-07-22T12:35:54,969 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(888): Instantiated table1,,1690029353811.da5ae8ac17989cb406336333d6113a9e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:35:54,970 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7296): checking encryption for da5ae8ac17989cb406336333d6113a9e 2023-07-22T12:35:54,970 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7299): checking classloading for da5ae8ac17989cb406336333d6113a9e 2023-07-22T12:35:54,971 INFO [StoreOpener-da5ae8ac17989cb406336333d6113a9e-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region da5ae8ac17989cb406336333d6113a9e 2023-07-22T12:35:54,972 INFO [StoreOpener-da5ae8ac17989cb406336333d6113a9e-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da5ae8ac17989cb406336333d6113a9e columnFamilyName f 2023-07-22T12:35:54,972 DEBUG [StoreOpener-da5ae8ac17989cb406336333d6113a9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:54,973 INFO [StoreOpener-da5ae8ac17989cb406336333d6113a9e-1 {}] regionserver.HStore(324): Store=da5ae8ac17989cb406336333d6113a9e/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:35:54,974 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/table1/da5ae8ac17989cb406336333d6113a9e 2023-07-22T12:35:54,974 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/table1/da5ae8ac17989cb406336333d6113a9e 2023-07-22T12:35:54,976 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1076): writing seq id for da5ae8ac17989cb406336333d6113a9e 2023-07-22T12:35:54,979 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/table1/da5ae8ac17989cb406336333d6113a9e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:35:54,979 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): Opened da5ae8ac17989cb406336333d6113a9e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11078220160, jitterRate=0.031739652156829834}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:35:54,980 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(992): Region open journal for da5ae8ac17989cb406336333d6113a9e: 2023-07-22T12:35:54,981 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2552): Post open deploy tasks for table1,,1690029353811.da5ae8ac17989cb406336333d6113a9e., pid=38, masterSystemTime=1690029354965 2023-07-22T12:35:54,983 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2579): Finished post open deploy task for table1,,1690029353811.da5ae8ac17989cb406336333d6113a9e. 2023-07-22T12:35:54,983 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(158): Opened table1,,1690029353811.da5ae8ac17989cb406336333d6113a9e. 2023-07-22T12:35:54,983 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=da5ae8ac17989cb406336333d6113a9e, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:54,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=38, resume processing ppid=37 2023-07-22T12:35:54,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=38, ppid=37, state=SUCCESS; OpenRegionProcedure da5ae8ac17989cb406336333d6113a9e, server=jenkins-hbase3.apache.org,38147,1690029314593 in 172 msec 2023-07-22T12:35:54,988 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=37, resume processing ppid=36 2023-07-22T12:35:54,988 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=37, ppid=36, state=SUCCESS; TransitRegionStateProcedure table=table1, region=da5ae8ac17989cb406336333d6113a9e, ASSIGN in 331 msec 2023-07-22T12:35:54,989 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:35:54,989 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029354989"}]},"ts":"1690029354989"} 2023-07-22T12:35:54,990 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLED in hbase:meta 2023-07-22T12:35:54,992 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:35:54,994 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=36, state=SUCCESS; CreateTableProcedure table=table1 in 1.1810 sec 2023-07-22T12:35:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-07-22T12:35:55,924 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: default:table1, procId: 36 completed 2023-07-22T12:35:55,930 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:38161/backupUT/backup_1690029338321/default/test-1690029330567/archive/data/default/test-1690029330567/e2f9453920ffaa2bfd87a3743468e74e from hbase tables test-1690029330567 to tables table1 2023-07-22T12:35:55,931 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.MapReduceRestoreJob(72): Restore test-1690029330567 into table1 2023-07-22T12:35:55,933 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/hbase-staging/bulk_output-default-table1-1690029355931 from hdfs://localhost:38161/backupUT/backup_1690029338321/default/test-1690029330567/archive/data/default/test-1690029330567/e2f9453920ffaa2bfd87a3743468e74e 2023-07-22T12:35:55,936 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x129f64e0 to 127.0.0.1:54609 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:35:55,942 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@642d05e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:35:55,943 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:35:55,944 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:51292, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:35:55,959 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-07-22T12:35:55,959 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table table1 2023-07-22T12:35:55,965 DEBUG [hconnection-0x462ffbf7-shared-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:35:55,967 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:34504, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:35:55,970 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for table1: 2023-07-22T12:35:55,971 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-07-22T12:35:55,971 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/hbase-staging/partitions_6c6a0334-2d96-4b97-890e-a1ebb1b11a59 2023-07-22T12:35:56,410 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:56,411 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:56,411 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:56,411 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:57,402 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/hadoop-643363378927823095.jar 2023-07-22T12:35:57,403 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:57,404 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:57,404 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:57,405 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:57,406 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:57,406 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:57,407 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:57,408 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:57,408 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:57,409 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:35:57,410 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:35:57,411 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:35:57,412 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:35:57,412 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:35:57,413 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:35:57,414 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:35:57,414 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:35:57,415 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:35:57,416 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:35:57,416 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:35:57,417 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:35:57,421 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:57,421 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:57,422 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:57,422 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:57,422 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:57,423 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:57,423 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:35:57,424 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: table1 2023-07-22T12:35:57,424 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] client.ConnectionImplementation(2095): Closing master protocol: MasterService 2023-07-22T12:35:57,424 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x129f64e0 to 127.0.0.1:54609 2023-07-22T12:35:57,424 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:35:57,424 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-07-22T12:35:57,424 WARN [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-07-22T12:35:57,425 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:35:57,430 WARN [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-07-22T12:35:57,437 WARN [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-07-22T12:35:57,522 WARN [Listener at jenkins-hbase3.apache.org/44413 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/mapreduce.cluster.local.dir/1690029357484/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:35:57,522 WARN [Listener at jenkins-hbase3.apache.org/44413 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/mapreduce.cluster.local.dir/1690029357484/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:35:57,598 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:38161/backupUT/backup_1690029338321/default/test-1690029330567/archive/data/default/test-1690029330567/e2f9453920ffaa2bfd87a3743468e74e/f/a663b1eb7c49421a9a8af7e19a9900bf 2023-07-22T12:35:57,603 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-07-22T12:35:57,607 DEBUG [LocalJobRunner Map Task Executor #0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a663b1eb7c49421a9a8af7e19a9900bf 2023-07-22T12:35:57,739 DEBUG [pool-318-thread-1 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2d2ff245 to 127.0.0.1:54609 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:35:57,747 DEBUG [pool-318-thread-1 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29b2e9c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:35:57,749 DEBUG [hconnection-0x2845b738-metaLookup-shared--pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:35:57,751 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:34506, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:35:57,753 DEBUG [pool-318-thread-1 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2d2ff245 to 127.0.0.1:54609 2023-07-22T12:35:57,753 DEBUG [pool-318-thread-1 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:35:57,755 DEBUG [pool-318-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-07-22T12:35:57,755 DEBUG [pool-318-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-07-22T12:35:57,769 DEBUG [pool-318-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/hbase-staging/bulk_output-default-table1-1690029355931/_temporary/0/_temporary/attempt_local1994730290_0002_r_000000_0/f/f1ac8788226e46a5bda10884f74fa3a3 is 37, key is row10/f:q1/1690029333239/Put/seqid=0 2023-07-22T12:35:57,771 WARN [Thread-987 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:38147] Specified favored nodes: [jenkins-hbase3.apache.org:38147] 2023-07-22T12:35:58,541 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'table1' 2023-07-22T12:35:58,583 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/hbase-staging/bulk_output-default-table1-1690029355931 2023-07-22T12:35:58,583 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x70a90217 to 127.0.0.1:54609 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:35:58,591 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31ff3b9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:35:58,592 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:35:58,594 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:34522, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:35:58,602 WARN [Listener at jenkins-hbase3.apache.org/44413 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/hbase-staging/bulk_output-default-table1-1690029355931/_SUCCESS 2023-07-22T12:35:58,607 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:35:58,608 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:51308, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:35:58,637 DEBUG [LoadIncrementalHFiles-0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: f1ac8788226e46a5bda10884f74fa3a3 2023-07-22T12:35:58,638 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/hbase-staging/bulk_output-default-table1-1690029355931/f/f1ac8788226e46a5bda10884f74fa3a3 first=Optional[row0] last=Optional[row98] 2023-07-22T12:35:58,659 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=table1,,1690029353811.da5ae8ac17989cb406336333d6113a9e., hostname=jenkins-hbase3.apache.org,38147,1690029314593, seqNum=2 for row with hfile group [{f,hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/hbase-staging/bulk_output-default-table1-1690029355931/f/f1ac8788226e46a5bda10884f74fa3a3}] 2023-07-22T12:35:58,678 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/hbase-staging/bulk_output-default-table1-1690029355931/f/f1ac8788226e46a5bda10884f74fa3a3 for inclusion in da5ae8ac17989cb406336333d6113a9e/f 2023-07-22T12:35:58,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-07-22T12:35:58,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] regionserver.HStore(625): Region bounds: first= last= 2023-07-22T12:35:58,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: f1ac8788226e46a5bda10884f74fa3a3 2023-07-22T12:35:58,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] regionserver.HRegion(2520): Flush status journal for da5ae8ac17989cb406336333d6113a9e: 2023-07-22T12:35:58,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/hbase-staging/bulk_output-default-table1-1690029355931/f/f1ac8788226e46a5bda10884f74fa3a3 to hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/staging/jenkins__table1__utbt7eer2bsl8lehkq0ue36aectcgaah028r7009mtj7mp7baksla4h0sqo9da4u/f/f1ac8788226e46a5bda10884f74fa3a3 2023-07-22T12:35:58,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/staging/jenkins__table1__utbt7eer2bsl8lehkq0ue36aectcgaah028r7009mtj7mp7baksla4h0sqo9da4u/f/f1ac8788226e46a5bda10884f74fa3a3 as hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/table1/da5ae8ac17989cb406336333d6113a9e/f/b7b48b760aef4c80b6e152bdce37d2c3_SeqId_4_ 2023-07-22T12:35:58,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5789547e to 127.0.0.1:54609 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:35:58,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3872dd58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:35:58,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:35:58,698 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:51320, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:35:58,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:35:58,704 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:34530, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:35:58,723 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] client.ConnectionImplementation(2095): Closing master protocol: MasterService 2023-07-22T12:35:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5789547e to 127.0.0.1:54609 2023-07-22T12:35:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:35:58,724 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/staging/jenkins__table1__utbt7eer2bsl8lehkq0ue36aectcgaah028r7009mtj7mp7baksla4h0sqo9da4u/f/f1ac8788226e46a5bda10884f74fa3a3 into da5ae8ac17989cb406336333d6113a9e/f as hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/table1/da5ae8ac17989cb406336333d6113a9e/f/b7b48b760aef4c80b6e152bdce37d2c3_SeqId_4_ - updating store file list. 2023-07-22T12:35:58,731 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/table1/da5ae8ac17989cb406336333d6113a9e/f/b7b48b760aef4c80b6e152bdce37d2c3_SeqId_4_ into da5ae8ac17989cb406336333d6113a9e/f 2023-07-22T12:35:58,731 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/staging/jenkins__table1__utbt7eer2bsl8lehkq0ue36aectcgaah028r7009mtj7mp7baksla4h0sqo9da4u/f/f1ac8788226e46a5bda10884f74fa3a3 into da5ae8ac17989cb406336333d6113a9e/f (new location: hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/table1/da5ae8ac17989cb406336333d6113a9e/f/b7b48b760aef4c80b6e152bdce37d2c3_SeqId_4_) 2023-07-22T12:35:58,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/staging/jenkins__table1__utbt7eer2bsl8lehkq0ue36aectcgaah028r7009mtj7mp7baksla4h0sqo9da4u/f/f1ac8788226e46a5bda10884f74fa3a3 2023-07-22T12:35:58,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x36346567 to 127.0.0.1:54609 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:35:58,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34665785, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:35:58,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:35:58,749 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:51324, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-07-22T12:35:58,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:35:58,753 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:34544, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-07-22T12:35:58,768 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] client.ConnectionImplementation(2095): Closing master protocol: MasterService 2023-07-22T12:35:58,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x36346567 to 127.0.0.1:54609 2023-07-22T12:35:58,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:35:58,779 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] client.ConnectionImplementation(2095): Closing master protocol: MasterService 2023-07-22T12:35:58,779 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x70a90217 to 127.0.0.1:54609 2023-07-22T12:35:58,779 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:35:58,780 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-07-22T12:35:58,780 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-07-22T12:35:58,780 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] impl.RestoreTablesClient(233): Backup: backup_1690029338321 hdfs://localhost:38161/backupUT/backup_1690029338321/default/test-1690029330567/ 2023-07-22T12:35:58,780 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-07-22T12:35:58,782 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] client.HBaseAdmin$18(963): Started disable of table1 2023-07-22T12:35:58,787 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable table1 2023-07-22T12:35:58,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] procedure2.ProcedureExecutor(1033): Stored pid=39, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=table1 2023-07-22T12:35:58,799 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029358799"}]},"ts":"1690029358799"} 2023-07-22T12:35:58,800 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLING in hbase:meta 2023-07-22T12:35:58,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=39 2023-07-22T12:35:58,803 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(293): Set table1 to state=DISABLING 2023-07-22T12:35:58,805 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=da5ae8ac17989cb406336333d6113a9e, UNASSIGN}] 2023-07-22T12:35:58,807 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=da5ae8ac17989cb406336333d6113a9e, UNASSIGN 2023-07-22T12:35:58,807 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=da5ae8ac17989cb406336333d6113a9e, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:35:58,809 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE; CloseRegionProcedure da5ae8ac17989cb406336333d6113a9e, server=jenkins-hbase3.apache.org,38147,1690029314593}] 2023-07-22T12:35:58,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=39 2023-07-22T12:35:58,970 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] handler.UnassignRegionHandler(119): Close da5ae8ac17989cb406336333d6113a9e 2023-07-22T12:35:58,970 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1668): Closing da5ae8ac17989cb406336333d6113a9e, disabling compactions & flushes 2023-07-22T12:35:58,970 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1690): Closing region table1,,1690029353811.da5ae8ac17989cb406336333d6113a9e. 2023-07-22T12:35:58,970 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1690029353811.da5ae8ac17989cb406336333d6113a9e. 2023-07-22T12:35:58,970 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1778): Acquired close lock on table1,,1690029353811.da5ae8ac17989cb406336333d6113a9e. after waiting 0 ms 2023-07-22T12:35:58,970 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1788): Updates disabled for region table1,,1690029353811.da5ae8ac17989cb406336333d6113a9e. 2023-07-22T12:35:58,973 DEBUG [StoreFileCloser-da5ae8ac17989cb406336333d6113a9e-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: b7b48b760aef4c80b6e152bdce37d2c3_SeqId_4_ 2023-07-22T12:35:58,978 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/table1/da5ae8ac17989cb406336333d6113a9e/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-07-22T12:35:58,979 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:35:58,979 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1909): Closed table1,,1690029353811.da5ae8ac17989cb406336333d6113a9e. 2023-07-22T12:35:58,979 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1622): Region close journal for da5ae8ac17989cb406336333d6113a9e: 2023-07-22T12:35:58,981 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] handler.UnassignRegionHandler(163): Closed da5ae8ac17989cb406336333d6113a9e 2023-07-22T12:35:58,981 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=da5ae8ac17989cb406336333d6113a9e, regionState=CLOSED 2023-07-22T12:35:58,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=41, resume processing ppid=40 2023-07-22T12:35:58,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=41, ppid=40, state=SUCCESS; CloseRegionProcedure da5ae8ac17989cb406336333d6113a9e, server=jenkins-hbase3.apache.org,38147,1690029314593 in 174 msec 2023-07-22T12:35:58,986 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=40, resume processing ppid=39 2023-07-22T12:35:58,986 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=40, ppid=39, state=SUCCESS; TransitRegionStateProcedure table=table1, region=da5ae8ac17989cb406336333d6113a9e, UNASSIGN in 180 msec 2023-07-22T12:35:58,987 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029358987"}]},"ts":"1690029358987"} 2023-07-22T12:35:58,988 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLED in hbase:meta 2023-07-22T12:35:58,990 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(305): Set table1 to state=DISABLED 2023-07-22T12:35:58,991 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=39, state=SUCCESS; DisableTableProcedure table=table1 in 200 msec 2023-07-22T12:35:59,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=39 2023-07-22T12:35:59,103 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] client.HBaseAdmin$TableFuture(3590): Operation: DISABLE, Table Name: default:table1, procId: 39 completed 2023-07-22T12:35:59,108 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.HMaster$5(2447): Client=jenkins//172.31.12.81 delete table1 2023-07-22T12:35:59,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] procedure2.ProcedureExecutor(1033): Stored pid=42, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=table1 2023-07-22T12:35:59,119 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=42, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-07-22T12:35:59,121 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=42, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=table1 2023-07-22T12:35:59,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-07-22T12:35:59,127 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/table1/da5ae8ac17989cb406336333d6113a9e 2023-07-22T12:35:59,131 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/table1/da5ae8ac17989cb406336333d6113a9e/f, FileablePath, hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/table1/da5ae8ac17989cb406336333d6113a9e/recovered.edits] 2023-07-22T12:35:59,138 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/table1/da5ae8ac17989cb406336333d6113a9e/f/b7b48b760aef4c80b6e152bdce37d2c3_SeqId_4_ to hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/archive/data/default/table1/da5ae8ac17989cb406336333d6113a9e/f/b7b48b760aef4c80b6e152bdce37d2c3_SeqId_4_ 2023-07-22T12:35:59,141 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/table1/da5ae8ac17989cb406336333d6113a9e/recovered.edits/6.seqid to hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/archive/data/default/table1/da5ae8ac17989cb406336333d6113a9e/recovered.edits/6.seqid 2023-07-22T12:35:59,142 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/table1/da5ae8ac17989cb406336333d6113a9e 2023-07-22T12:35:59,142 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(293): Archived table1 regions 2023-07-22T12:35:59,144 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=42, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=table1 2023-07-22T12:35:59,154 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of table1 from hbase:meta 2023-07-22T12:35:59,156 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(387): Removing 'table1' descriptor. 2023-07-22T12:35:59,157 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=42, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=table1 2023-07-22T12:35:59,157 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(377): Removing 'table1' from region states. 2023-07-22T12:35:59,157 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1,,1690029353811.da5ae8ac17989cb406336333d6113a9e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1690029359157"}]},"ts":"9223372036854775807"} 2023-07-22T12:35:59,159 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-07-22T12:35:59,159 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => da5ae8ac17989cb406336333d6113a9e, NAME => 'table1,,1690029353811.da5ae8ac17989cb406336333d6113a9e.', STARTKEY => '', ENDKEY => ''}] 2023-07-22T12:35:59,160 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(381): Marking 'table1' as deleted. 2023-07-22T12:35:59,160 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1690029359160"}]},"ts":"9223372036854775807"} 2023-07-22T12:35:59,161 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1678): Deleted table table1 state from META 2023-07-22T12:35:59,163 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(127): Finished pid=42, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-07-22T12:35:59,164 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=42, state=SUCCESS; DeleteTableProcedure table=table1 in 53 msec 2023-07-22T12:35:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-07-22T12:35:59,223 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] client.HBaseAdmin$TableFuture(3590): Operation: DELETE, Table Name: default:table1, procId: 42 completed 2023-07-22T12:35:59,247 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.ResourceChecker(175): after: backup.TestRemoteRestore#testFullRestoreRemote Thread=831 (was 816) Potentially hanging thread: hconnection-0x552edb61-shared-pool-8 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,38147,1690029314593)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/cluster_81664056-bd3e-8e52-f121-db92c3ef923e/dfs/data/data1/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: (jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-972608796_16 at /127.0.0.1:55334 [Receiving block BP-1697032271-172.31.12.81-1690029308490:blk_1073741857_1033] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_344205816_16 at /127.0.0.1:57608 [Waiting for operation #8] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Client (2082644736) connection to localhost/127.0.0.1:38161 from jenkins.hfs.1 java.lang.Object.wait(Native Method) org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1053) org.apache.hadoop.ipc.Client$Connection.run(Client.java:1097) Potentially hanging thread: HFileArchiver-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x552edb61-shared-pool-6 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1697032271-172.31.12.81-1690029308490:blk_1073741858_1034, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-972608796_16 at /127.0.0.1:55338 [Receiving block BP-1697032271-172.31.12.81-1690029308490:blk_1073741858_1034] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1697032271-172.31.12.81-1690029308490:blk_1073741857_1033, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x552edb61-shared-pool-11 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x552edb61-shared-pool-9 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2126397842_16 at /127.0.0.1:59190 [Waiting for operation #4] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/cluster_81664056-bd3e-8e52-f121-db92c3ef923e/dfs/data/data2/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x552edb61-shared-pool-7 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x552edb61-shared-pool-10 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: member: 'jenkins-hbase3.apache.org,38147,1690029314593' subprocedure-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS-EventLoopGroup-5-3 org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:209) org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:202) org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:306) org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:363) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=903 (was 878) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=90 (was 104), ProcessCount=169 (was 169), AvailableMemoryMB=16459 (was 16717) 2023-07-22T12:35:59,247 WARN [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.ResourceChecker(130): Thread=831 is superior to 500 2023-07-22T12:35:59,267 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.ResourceChecker(147): before: backup.TestRemoteRestore#testFullRestoreRemoteWithAlternateRestoreOutputDir Thread=831, OpenFileDescriptor=903, MaxFileDescriptor=60000, SystemLoadAverage=90, ProcessCount=169, AvailableMemoryMB=16457 2023-07-22T12:35:59,267 WARN [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.ResourceChecker(130): Thread=831 is superior to 500 2023-07-22T12:35:59,267 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] backup.TestRemoteRestore(91): test remote full backup on a single table with alternate restore output dir 2023-07-22T12:35:59,267 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5459addd to 127.0.0.1:54609 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:35:59,273 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6541b5fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:35:59,276 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:35:59,278 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:34554, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:35:59,279 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:35:59,281 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:51326, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:35:59,294 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:35:59,301 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:35:59,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029359301 (current time:1690029359301). 2023-07-22T12:35:59,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:35:59,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:35:59,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x09355251 to 127.0.0.1:54609 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:35:59,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@568719e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:35:59,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:35:59,311 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:34564, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:35:59,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x09355251 to 127.0.0.1:54609 2023-07-22T12:35:59,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:35:59,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:35:59,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] procedure2.ProcedureExecutor(1033): Stored pid=43, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:35:59,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 43 2023-07-22T12:35:59,315 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:35:59,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=43 2023-07-22T12:35:59,316 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:35:59,318 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:35:59,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=43 2023-07-22T12:35:59,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=43 2023-07-22T12:35:59,725 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:35:59,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE; SnapshotRegionProcedure 5c9301698cb41dfb7fd9be1c9c6a1f19}] 2023-07-22T12:35:59,728 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=44, ppid=43, state=RUNNABLE; SnapshotRegionProcedure 5c9301698cb41dfb7fd9be1c9c6a1f19 2023-07-22T12:35:59,740 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-07-22T12:35:59,740 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-07-22T12:35:59,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38147 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=44 2023-07-22T12:35:59,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690029338484.5c9301698cb41dfb7fd9be1c9c6a1f19. 2023-07-22T12:35:59,880 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegion(2819): Flushing 5c9301698cb41dfb7fd9be1c9c6a1f19 2/2 column families, dataSize=1.08 KB heapSize=2.23 KB 2023-07-22T12:35:59,896 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/.tmp/meta/e91b686b0ef14d2ca9fa2273c2792e2a is 169, key is trslm:hdfs://localhost:38161/backupUT\x00test-1690029330567/meta:log-roll-map/1690029353250/Put/seqid=0 2023-07-22T12:35:59,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=43 2023-07-22T12:36:00,301 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=526 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/.tmp/meta/e91b686b0ef14d2ca9fa2273c2792e2a 2023-07-22T12:36:00,307 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: e91b686b0ef14d2ca9fa2273c2792e2a 2023-07-22T12:36:00,310 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/.tmp/session/eb02733a331149f6a13d862744b427bc is 310, key is session:backup_1690029338321/session:context/1690029353737/Put/seqid=0 2023-07-22T12:36:00,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=43 2023-07-22T12:36:00,618 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:36:00,715 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=579 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/.tmp/session/eb02733a331149f6a13d862744b427bc 2023-07-22T12:36:00,721 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: eb02733a331149f6a13d862744b427bc 2023-07-22T12:36:00,722 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/.tmp/meta/e91b686b0ef14d2ca9fa2273c2792e2a as hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/meta/e91b686b0ef14d2ca9fa2273c2792e2a 2023-07-22T12:36:00,728 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/meta/e91b686b0ef14d2ca9fa2273c2792e2a, entries=4, sequenceid=17, filesize=5.5 K 2023-07-22T12:36:00,729 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/.tmp/session/eb02733a331149f6a13d862744b427bc as hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/session/eb02733a331149f6a13d862744b427bc 2023-07-22T12:36:00,735 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/session/eb02733a331149f6a13d862744b427bc, entries=2, sequenceid=17, filesize=5.3 K 2023-07-22T12:36:00,736 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegion(3022): Finished flush of dataSize ~1.08 KB/1105, heapSize ~2.20 KB/2256, currentSize=0 B/0 for 5c9301698cb41dfb7fd9be1c9c6a1f19 in 856ms, sequenceid=17, compaction requested=false 2023-07-22T12:36:00,736 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegion(2520): Flush status journal for 5c9301698cb41dfb7fd9be1c9c6a1f19: 2023-07-22T12:36:00,736 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690029338484.5c9301698cb41dfb7fd9be1c9c6a1f19. for snapshot_backup_system completed. 2023-07-22T12:36:00,737 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690029338484.5c9301698cb41dfb7fd9be1c9c6a1f19.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:36:00,737 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:36:00,737 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/meta/e91b686b0ef14d2ca9fa2273c2792e2a] hfiles 2023-07-22T12:36:00,737 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/meta/e91b686b0ef14d2ca9fa2273c2792e2a for snapshot=snapshot_backup_system 2023-07-22T12:36:00,737 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/session/d3f88af37ddf42418c7fa151a0dd871a, hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/session/eb02733a331149f6a13d862744b427bc] hfiles 2023-07-22T12:36:00,737 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/session/d3f88af37ddf42418c7fa151a0dd871a for snapshot=snapshot_backup_system 2023-07-22T12:36:00,737 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/session/eb02733a331149f6a13d862744b427bc for snapshot=snapshot_backup_system 2023-07-22T12:36:01,142 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690029338484.5c9301698cb41dfb7fd9be1c9c6a1f19. 2023-07-22T12:36:01,143 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=44 2023-07-22T12:36:01,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.HMaster(3994): Remote procedure done, pid=44 2023-07-22T12:36:01,143 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 5c9301698cb41dfb7fd9be1c9c6a1f19 2023-07-22T12:36:01,145 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=44, ppid=43, state=RUNNABLE; SnapshotRegionProcedure 5c9301698cb41dfb7fd9be1c9c6a1f19 2023-07-22T12:36:01,147 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=44, resume processing ppid=43 2023-07-22T12:36:01,147 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:36:01,148 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=44, ppid=43, state=SUCCESS; SnapshotRegionProcedure 5c9301698cb41dfb7fd9be1c9c6a1f19 in 1.4210 sec 2023-07-22T12:36:01,148 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:36:01,149 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:36:01,149 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:36:01,150 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:36:01,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=43 2023-07-22T12:36:01,561 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:36:01,568 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:36:01,568 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:36:02,387 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:36:02,387 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 43 2023-07-22T12:36:02,388 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=43, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0740 sec 2023-07-22T12:36:03,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=43 2023-07-22T12:36:03,423 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: backup:system, procId: 43 completed 2023-07-22T12:36:03,423 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] impl.TableBackupClient(115): Backup backup_1690029359273 started at 1690029363423. 2023-07-22T12:36:03,424 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.TableBackupClient(119): Backup session backup_1690029359273 has been started. 2023-07-22T12:36:03,430 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-07-22T12:36:03,431 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-07-22T12:36:03,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-07-22T12:36:03,431 INFO [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-07-22T12:36:03,431 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-07-22T12:36:03,432 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-07-22T12:36:03,432 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-07-22T12:36:03,433 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-07-22T12:36:03,433 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:36:03,434 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-07-22T12:36:03,434 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:36:03,434 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:36:03,434 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:36:03,434 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:36:03,434 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-07-22T12:36:03,435 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-07-22T12:36:03,435 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-07-22T12:36:03,435 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-07-22T12:36:03,435 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:36:03,435 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-07-22T12:36:03,435 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-07-22T12:36:03,435 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-07-22T12:36:03,436 DEBUG [member: 'jenkins-hbase3.apache.org,38147,1690029314593' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-07-22T12:36:03,436 DEBUG [member: 'jenkins-hbase3.apache.org,38147,1690029314593' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-07-22T12:36:03,436 DEBUG [member: 'jenkins-hbase3.apache.org,38147,1690029314593' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-07-22T12:36:03,436 DEBUG [member: 'jenkins-hbase3.apache.org,38147,1690029314593' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-07-22T12:36:03,436 DEBUG [member: 'jenkins-hbase3.apache.org,38147,1690029314593' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,38147,1690029314593' joining acquired barrier for procedure (rolllog) in zk 2023-07-22T12:36:03,438 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:36:03,438 DEBUG [member: 'jenkins-hbase3.apache.org,38147,1690029314593' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-07-22T12:36:03,438 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:36:03,438 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:36:03,438 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:36:03,438 DEBUG [member: 'jenkins-hbase3.apache.org,38147,1690029314593' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-07-22T12:36:03,438 DEBUG [member: 'jenkins-hbase3.apache.org,38147,1690029314593' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-07-22T12:36:03,438 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:36:03,439 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:36:03,439 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:36:03,439 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:36:03,439 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:36:03,440 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,38147,1690029314593' joining acquired barrier for procedure 'rolllog' on coordinator 2023-07-22T12:36:03,440 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@42ca68c0[Count = 0] remaining members to acquire global barrier 2023-07-22T12:36:03,440 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-07-22T12:36:03,440 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-07-22T12:36:03,442 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:36:03,442 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-07-22T12:36:03,442 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-07-22T12:36:03,442 DEBUG [member: 'jenkins-hbase3.apache.org,38147,1690029314593' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-07-22T12:36:03,442 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:36:03,442 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-07-22T12:36:03,442 DEBUG [member: 'jenkins-hbase3.apache.org,38147,1690029314593' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-07-22T12:36:03,442 DEBUG [rs(jenkins-hbase3.apache.org,38147,1690029314593)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:36:03,442 INFO [rs(jenkins-hbase3.apache.org,38147,1690029314593)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1690029346000 highest: 1690029346000 on jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:36:03,442 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C38147%2C1690029314593.meta:.meta(num 1690029345977) roll requested 2023-07-22T12:36:03,454 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/WALs/jenkins-hbase3.apache.org,38147,1690029314593/jenkins-hbase3.apache.org%2C38147%2C1690029314593.meta.1690029363442.meta, exclude list is [], retry=0 2023-07-22T12:36:03,458 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38553,DS-c76e90c8-39fa-4f11-ba17-53eb00da3429,DISK] 2023-07-22T12:36:03,459 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/WALs/jenkins-hbase3.apache.org,38147,1690029314593/jenkins-hbase3.apache.org%2C38147%2C1690029314593.meta.1690029345977.meta with entries=12, filesize=2.73 KB; new WAL /user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/WALs/jenkins-hbase3.apache.org,38147,1690029314593/jenkins-hbase3.apache.org%2C38147%2C1690029314593.meta.1690029363442.meta 2023-07-22T12:36:03,459 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38553,DS-c76e90c8-39fa-4f11-ba17-53eb00da3429,DISK]] 2023-07-22T12:36:03,460 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/WALs/jenkins-hbase3.apache.org,38147,1690029314593/jenkins-hbase3.apache.org%2C38147%2C1690029314593.meta.1690029345977.meta is not closed yet, will try archiving it next time 2023-07-22T12:36:03,460 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C38147%2C1690029314593:(num 1690029346000) roll requested 2023-07-22T12:36:03,471 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/WALs/jenkins-hbase3.apache.org,38147,1690029314593/jenkins-hbase3.apache.org%2C38147%2C1690029314593.1690029363460, exclude list is [], retry=0 2023-07-22T12:36:03,477 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38553,DS-c76e90c8-39fa-4f11-ba17-53eb00da3429,DISK] 2023-07-22T12:36:03,479 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/WALs/jenkins-hbase3.apache.org,38147,1690029314593/jenkins-hbase3.apache.org%2C38147%2C1690029314593.1690029346000 with entries=15, filesize=3.47 KB; new WAL /user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/WALs/jenkins-hbase3.apache.org,38147,1690029314593/jenkins-hbase3.apache.org%2C38147%2C1690029314593.1690029363460 2023-07-22T12:36:03,479 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38553,DS-c76e90c8-39fa-4f11-ba17-53eb00da3429,DISK]] 2023-07-22T12:36:03,479 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/WALs/jenkins-hbase3.apache.org,38147,1690029314593/jenkins-hbase3.apache.org%2C38147%2C1690029314593.1690029346000 is not closed yet, will try archiving it next time 2023-07-22T12:36:03,482 DEBUG [rs(jenkins-hbase3.apache.org,38147,1690029314593)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-07-22T12:36:03,482 INFO [rs(jenkins-hbase3.apache.org,38147,1690029314593)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1690029363460 on jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:36:03,492 DEBUG [member: 'jenkins-hbase3.apache.org,38147,1690029314593' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-07-22T12:36:03,492 DEBUG [member: 'jenkins-hbase3.apache.org,38147,1690029314593' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,38147,1690029314593' in zk 2023-07-22T12:36:03,493 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:36:03,494 DEBUG [member: 'jenkins-hbase3.apache.org,38147,1690029314593' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-07-22T12:36:03,494 DEBUG [member: 'jenkins-hbase3.apache.org,38147,1690029314593' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:36:03,494 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:36:03,494 DEBUG [member: 'jenkins-hbase3.apache.org,38147,1690029314593' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-07-22T12:36:03,494 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:36:03,495 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:36:03,495 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:36:03,495 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:36:03,496 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:36:03,496 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:36:03,496 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:36:03,496 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:36:03,497 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:36:03,497 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,38147,1690029314593': 2023-07-22T12:36:03,497 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,38147,1690029314593' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-07-22T12:36:03,497 INFO [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-07-22T12:36:03,497 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-07-22T12:36:03,497 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-07-22T12:36:03,498 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-07-22T12:36:03,498 INFO [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-07-22T12:36:03,499 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:36:03,499 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:36:03,499 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-07-22T12:36:03,499 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-07-22T12:36:03,499 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-07-22T12:36:03,499 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:36:03,499 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:36:03,499 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:36:03,500 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:36:03,500 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:36:03,500 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:36:03,500 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:36:03,500 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:36:03,500 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:36:03,500 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:36:03,501 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:36:03,501 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:36:03,501 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:36:03,501 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:36:03,502 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:36:03,502 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:36:03,504 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:36:03,504 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-07-22T12:36:03,504 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-07-22T12:36:03,504 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-07-22T12:36:03,504 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:36:03,504 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:36:03,504 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:36:03,504 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-07-22T12:36:03,504 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:36:03,504 DEBUG [(jenkins-hbase3.apache.org,46137,1690029311532)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:36:03,504 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:36:03,504 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-07-22T12:36:03,505 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:36:03,505 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-07-22T12:36:03,505 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2023-07-22T12:36:03,505 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] client.HBaseAdmin(2751): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-07-22T12:36:03,505 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:36:03,505 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] client.HBaseAdmin(2759): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-07-22T12:36:03,505 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:36:03,605 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] client.HBaseAdmin(2765): Getting current status of procedure from master... 2023-07-22T12:36:03,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-07-22T12:36:03,609 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1690029363608_default_test-1690029330567 table=test-1690029330567 type=FLUSH ttl=0 } 2023-07-22T12:36:03,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029363609 (current time:1690029363609). 2023-07-22T12:36:03,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:36:03,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1690029363608_default_test-1690029330567 VERSION not specified, setting to 2 2023-07-22T12:36:03,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x247eb34e to 127.0.0.1:54609 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:36:03,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@280bacd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:36:03,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:36:03,626 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:55018, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:36:03,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x247eb34e to 127.0.0.1:54609 2023-07-22T12:36:03,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:36:03,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:36:03,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] procedure2.ProcedureExecutor(1033): Stored pid=45, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1690029363608_default_test-1690029330567 table=test-1690029330567 type=FLUSH ttl=0 } 2023-07-22T12:36:03,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1690029363608_default_test-1690029330567 table=test-1690029330567 type=FLUSH ttl=0 }, snapshot procedure id = 45 2023-07-22T12:36:03,631 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1690029363608_default_test-1690029330567 table=test-1690029330567 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:36:03,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=45 2023-07-22T12:36:03,632 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1690029363608_default_test-1690029330567 table=test-1690029330567 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:36:03,633 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1690029363608_default_test-1690029330567 table=test-1690029330567 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:36:03,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=45 2023-07-22T12:36:03,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=45 2023-07-22T12:36:04,039 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1690029363608_default_test-1690029330567 table=test-1690029330567 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:36:04,040 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE; SnapshotRegionProcedure e2f9453920ffaa2bfd87a3743468e74e}] 2023-07-22T12:36:04,042 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=46, ppid=45, state=RUNNABLE; SnapshotRegionProcedure e2f9453920ffaa2bfd87a3743468e74e 2023-07-22T12:36:04,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38147 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=46 2023-07-22T12:36:04,193 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1690029330567,,1690029331075.e2f9453920ffaa2bfd87a3743468e74e. 2023-07-22T12:36:04,194 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] regionserver.HRegion(2520): Flush status journal for e2f9453920ffaa2bfd87a3743468e74e: 2023-07-22T12:36:04,194 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1690029330567,,1690029331075.e2f9453920ffaa2bfd87a3743468e74e. for snapshot_1690029363608_default_test-1690029330567 completed. 2023-07-22T12:36:04,194 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] snapshot.SnapshotManifest(240): Storing 'test-1690029330567,,1690029331075.e2f9453920ffaa2bfd87a3743468e74e.' region-info for snapshot=snapshot_1690029363608_default_test-1690029330567 2023-07-22T12:36:04,194 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:36:04,194 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/test-1690029330567/e2f9453920ffaa2bfd87a3743468e74e/f/a663b1eb7c49421a9a8af7e19a9900bf] hfiles 2023-07-22T12:36:04,194 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/test-1690029330567/e2f9453920ffaa2bfd87a3743468e74e/f/a663b1eb7c49421a9a8af7e19a9900bf for snapshot=snapshot_1690029363608_default_test-1690029330567 2023-07-22T12:36:04,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=45 2023-07-22T12:36:04,600 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1690029330567,,1690029331075.e2f9453920ffaa2bfd87a3743468e74e. 2023-07-22T12:36:04,600 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=46 2023-07-22T12:36:04,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.HMaster(3994): Remote procedure done, pid=46 2023-07-22T12:36:04,601 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1690029363608_default_test-1690029330567 on region e2f9453920ffaa2bfd87a3743468e74e 2023-07-22T12:36:04,603 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=46, ppid=45, state=RUNNABLE; SnapshotRegionProcedure e2f9453920ffaa2bfd87a3743468e74e 2023-07-22T12:36:04,605 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=46, resume processing ppid=45 2023-07-22T12:36:04,605 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1690029363608_default_test-1690029330567 table=test-1690029330567 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:36:04,605 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=46, ppid=45, state=SUCCESS; SnapshotRegionProcedure e2f9453920ffaa2bfd87a3743468e74e in 564 msec 2023-07-22T12:36:04,605 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1690029363608_default_test-1690029330567 table=test-1690029330567 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:36:04,606 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1690029363608_default_test-1690029330567 table=test-1690029330567 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:36:04,606 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1690029363608_default_test-1690029330567 2023-07-22T12:36:04,607 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/.hbase-snapshot/.tmp/snapshot_1690029363608_default_test-1690029330567 2023-07-22T12:36:04,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=45 2023-07-22T12:36:05,017 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1690029363608_default_test-1690029330567 table=test-1690029330567 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:36:05,023 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1690029363608_default_test-1690029330567 table=test-1690029330567 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:36:05,023 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/.hbase-snapshot/.tmp/snapshot_1690029363608_default_test-1690029330567 to hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/.hbase-snapshot/snapshot_1690029363608_default_test-1690029330567 2023-07-22T12:36:05,135 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-07-22T12:36:05,135 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-07-22T12:36:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=45 2023-07-22T12:36:05,841 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1690029363608_default_test-1690029330567 table=test-1690029330567 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:36:05,841 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1690029363608_default_test-1690029330567 table=test-1690029330567 type=FLUSH ttl=0 }, snapshot procedure id = 45 2023-07-22T12:36:05,842 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=45, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1690029363608_default_test-1690029330567 table=test-1690029330567 type=FLUSH ttl=0 } in 2.2140 sec 2023-07-22T12:36:07,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=45 2023-07-22T12:36:07,738 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: default:test-1690029330567, procId: 45 completed 2023-07-22T12:36:07,738 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1690029359273 2023-07-22T12:36:07,738 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-07-22T12:36:07,738 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-07-22T12:36:07,738 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1690029359273_test-1690029330567 2023-07-22T12:36:07,738 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1690029363608_default_test-1690029330567 to hdfs://localhost:38161/backupUT/backup_1690029359273/default/test-1690029330567/ 2023-07-22T12:36:07,738 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-07-22T12:36:07,765 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:38685, inputRoot=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f 2023-07-22T12:36:07,765 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,46137,1690029311532_636562561_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:38161/backupUT/backup_1690029359273/default/test-1690029330567, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:38161/backupUT/backup_1690029359273/default/test-1690029330567/.hbase-snapshot/.tmp/snapshot_1690029363608_default_test-1690029330567 2023-07-22T12:36:07,765 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:38685, inputRoot=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f, snapshotDir=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/.hbase-snapshot/snapshot_1690029363608_default_test-1690029330567. 2023-07-22T12:36:07,777 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/.hbase-snapshot/snapshot_1690029363608_default_test-1690029330567 to hdfs://localhost:38161/backupUT/backup_1690029359273/default/test-1690029330567/.hbase-snapshot/.tmp/snapshot_1690029363608_default_test-1690029330567 2023-07-22T12:36:08,190 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:36:08,191 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:36:08,191 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:36:08,192 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:36:09,194 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/hadoop-5945816446125012781.jar 2023-07-22T12:36:09,194 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:36:09,195 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:36:09,195 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:36:09,196 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:36:09,196 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:36:09,196 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:36:09,197 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:36:09,197 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:36:09,198 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:36:09,198 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:36:09,199 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:36:09,199 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:36:09,200 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:36:09,200 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:36:09,201 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:36:09,201 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:36:09,202 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:36:09,202 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:36:09,202 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:36:09,203 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:36:09,203 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:36:09,204 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:36:09,204 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:36:09,205 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:36:09,205 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:36:09,206 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:36:09,206 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:36:09,207 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:36:09,235 WARN [Listener at jenkins-hbase3.apache.org/44413 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-07-22T12:36:09,259 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1690029363608_default_test-1690029330567' hfile list 2023-07-22T12:36:09,262 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-07-22T12:36:09,351 WARN [Listener at jenkins-hbase3.apache.org/44413 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/mapreduce.cluster.local.dir/1690029369310/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:36:09,352 WARN [Listener at jenkins-hbase3.apache.org/44413 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/mapreduce.cluster.local.dir/1690029369310/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:36:09,406 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-07-22T12:36:09,431 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/test-1690029330567/e2f9453920ffaa2bfd87a3743468e74e/f/a663b1eb7c49421a9a8af7e19a9900bf output=hdfs://localhost:38161/backupUT/backup_1690029359273/default/test-1690029330567/archive/data/default/test-1690029330567/e2f9453920ffaa2bfd87a3743468e74e/f/a663b1eb7c49421a9a8af7e19a9900bf 2023-07-22T12:36:09,431 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 8.007M/sec 2023-07-22T12:36:10,389 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-07-22T12:36:10,390 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-07-22T12:36:10,398 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1690029363608_default_test-1690029330567 2023-07-22T12:36:10,398 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1690029363608_default_test-1690029330567 finished. 2023-07-22T12:36:10,398 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.BackupSystemTable(1014): test-1690029330567 2023-07-22T12:36:10,405 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029359273 2023-07-22T12:36:10,405 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-07-22T12:36:10,810 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:38161/backupUT/backup_1690029359273/.backup.manifest 2023-07-22T12:36:10,810 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.TableBackupClient(358): Backup backup_1690029359273 finished: type=FULL,tablelist=test-1690029330567,targetRootDir=hdfs://localhost:38161/backupUT,startts=1690029363423,completets=1690029370405,bytescopied=0 2023-07-22T12:36:10,810 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-07-22T12:36:10,811 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1690029363608_default_test-1690029330567 2023-07-22T12:36:10,812 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1690029363608_default_test-1690029330567" 2023-07-22T12:36:10,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1690029363608_default_test-1690029330567 2023-07-22T12:36:10,815 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1690029363608_default_test-1690029330567 for backup backup_1690029359273 succeeded. 2023-07-22T12:36:10,816 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:36:10,820 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:36:10,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:36:10,823 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:36:10,825 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:36:10,832 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] impl.TableBackupClient(376): Backup backup_1690029359273 completed. 2023-07-22T12:36:10,832 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] client.ConnectionImplementation(2095): Closing master protocol: MasterService 2023-07-22T12:36:10,832 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5459addd to 127.0.0.1:54609 2023-07-22T12:36:10,832 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:36:10,832 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] backup.TestRemoteRestore(94): backup complete 2023-07-22T12:36:10,844 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/hadoop.log.dir so I do NOT create it in target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8 2023-07-22T12:36:10,844 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/hadoop.tmp.dir so I do NOT create it in target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8 2023-07-22T12:36:10,844 WARN [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.HBaseTestingUtility(455): hadoop.tmp.dir property value differs in configuration and system: Configuration=/tmp/hadoop-jenkins while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/hadoop.tmp.dir Erasing configuration value by system value. 2023-07-22T12:36:10,844 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8 2023-07-22T12:36:10,844 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/cluster_c16b38c6-f9ae-d092-5fc7-64996592badb, deleteOnExit=true 2023-07-22T12:36:10,844 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/test.cache.data in system properties and HBase conf 2023-07-22T12:36:10,844 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/hadoop.tmp.dir in system properties and HBase conf 2023-07-22T12:36:10,844 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/hadoop.log.dir in system properties and HBase conf 2023-07-22T12:36:10,844 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/mapreduce.cluster.local.dir in system properties and HBase conf 2023-07-22T12:36:10,844 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-07-22T12:36:10,845 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-07-22T12:36:10,845 DEBUG [Listener at jenkins-hbase3.apache.org/44413 {}] fs.HFileSystem(308): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-07-22T12:36:10,845 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-07-22T12:36:10,845 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-07-22T12:36:10,845 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-07-22T12:36:10,845 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-07-22T12:36:10,845 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-07-22T12:36:10,845 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-07-22T12:36:10,845 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-07-22T12:36:10,846 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/dfs.journalnode.edits.dir in system properties and HBase conf 2023-07-22T12:36:10,846 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-07-22T12:36:10,846 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/nfs.dump.dir in system properties and HBase conf 2023-07-22T12:36:10,846 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/java.io.tmpdir in system properties and HBase conf 2023-07-22T12:36:10,846 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/dfs.journalnode.edits.dir in system properties and HBase conf 2023-07-22T12:36:10,846 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-07-22T12:36:10,846 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-07-22T12:36:10,859 WARN [Listener at jenkins-hbase3.apache.org/44413 {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-07-22T12:36:10,859 WARN [Listener at jenkins-hbase3.apache.org/44413 {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-07-22T12:36:10,909 WARN [Listener at jenkins-hbase3.apache.org/44413 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:36:10,911 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:36:10,917 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/java.io.tmpdir/Jetty_localhost_34053_hdfs____n1u06v/webapp 2023-07-22T12:36:11,017 INFO [Listener at jenkins-hbase3.apache.org/44413 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:34053 2023-07-22T12:36:11,030 WARN [Listener at jenkins-hbase3.apache.org/44413 {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-07-22T12:36:11,030 WARN [Listener at jenkins-hbase3.apache.org/44413 {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-07-22T12:36:11,079 WARN [Listener at localhost/40741 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-07-22T12:36:11,102 WARN [Listener at localhost/40741 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-07-22T12:36:11,105 WARN [Listener at localhost/40741 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:36:11,106 INFO [Listener at localhost/40741 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:36:11,114 INFO [Listener at localhost/40741 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/java.io.tmpdir/Jetty_localhost_41673_datanode____a69pwc/webapp 2023-07-22T12:36:11,220 INFO [Listener at localhost/40741 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:41673 2023-07-22T12:36:11,236 WARN [Listener at localhost/44903 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-07-22T12:36:11,252 WARN [Listener at localhost/44903 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-07-22T12:36:11,253 WARN [Listener at localhost/44903 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:36:11,255 INFO [Listener at localhost/44903 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:36:11,259 INFO [Listener at localhost/44903 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/java.io.tmpdir/Jetty_localhost_43003_datanode____.qqwjb9/webapp 2023-07-22T12:36:11,314 WARN [Thread-1147 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-07-22T12:36:11,347 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xb699b3dd62e46ea0: Processing first storage report for DS-4e855d94-c9e2-4b5d-85c3-14df2dcbecff from datanode c271ad2c-37f7-4144-a5b2-d4a88cf3cd80 2023-07-22T12:36:11,347 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xb699b3dd62e46ea0: from storage DS-4e855d94-c9e2-4b5d-85c3-14df2dcbecff node DatanodeRegistration(127.0.0.1:38259, datanodeUuid=c271ad2c-37f7-4144-a5b2-d4a88cf3cd80, infoPort=37293, infoSecurePort=0, ipcPort=44903, storageInfo=lv=-57;cid=testClusterID;nsid=1660746884;c=1690029370861), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2023-07-22T12:36:11,347 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xb699b3dd62e46ea0: Processing first storage report for DS-59ff5e23-3882-4d16-8028-7fa707192f56 from datanode c271ad2c-37f7-4144-a5b2-d4a88cf3cd80 2023-07-22T12:36:11,347 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xb699b3dd62e46ea0: from storage DS-59ff5e23-3882-4d16-8028-7fa707192f56 node DatanodeRegistration(127.0.0.1:38259, datanodeUuid=c271ad2c-37f7-4144-a5b2-d4a88cf3cd80, infoPort=37293, infoSecurePort=0, ipcPort=44903, storageInfo=lv=-57;cid=testClusterID;nsid=1660746884;c=1690029370861), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-07-22T12:36:11,370 INFO [Listener at localhost/44903 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:43003 2023-07-22T12:36:11,387 WARN [Listener at localhost/41929 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-07-22T12:36:11,422 WARN [Listener at localhost/41929 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-07-22T12:36:11,424 WARN [Listener at localhost/41929 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:36:11,425 INFO [Listener at localhost/41929 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:36:11,431 INFO [Listener at localhost/41929 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/java.io.tmpdir/Jetty_localhost_40717_datanode____39wxe0/webapp 2023-07-22T12:36:11,476 WARN [Thread-1182 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-07-22T12:36:11,501 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x93ccc0fbdbd672a: Processing first storage report for DS-8f1db54a-2085-46aa-b423-325a011919b8 from datanode 460c43eb-c664-4701-8700-99018726209f 2023-07-22T12:36:11,501 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x93ccc0fbdbd672a: from storage DS-8f1db54a-2085-46aa-b423-325a011919b8 node DatanodeRegistration(127.0.0.1:35757, datanodeUuid=460c43eb-c664-4701-8700-99018726209f, infoPort=40679, infoSecurePort=0, ipcPort=41929, storageInfo=lv=-57;cid=testClusterID;nsid=1660746884;c=1690029370861), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2023-07-22T12:36:11,501 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x93ccc0fbdbd672a: Processing first storage report for DS-35cfcf5a-43cf-4d56-bd61-de99afd2b1b3 from datanode 460c43eb-c664-4701-8700-99018726209f 2023-07-22T12:36:11,501 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x93ccc0fbdbd672a: from storage DS-35cfcf5a-43cf-4d56-bd61-de99afd2b1b3 node DatanodeRegistration(127.0.0.1:35757, datanodeUuid=460c43eb-c664-4701-8700-99018726209f, infoPort=40679, infoSecurePort=0, ipcPort=41929, storageInfo=lv=-57;cid=testClusterID;nsid=1660746884;c=1690029370861), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-07-22T12:36:11,540 INFO [Listener at localhost/41929 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:40717 2023-07-22T12:36:11,559 WARN [Listener at localhost/33727 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-07-22T12:36:11,627 WARN [Thread-1216 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-07-22T12:36:11,650 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xb81e3c9da16312bd: Processing first storage report for DS-571338ac-e020-4cfa-a70a-55dc3041be92 from datanode de54bb37-b730-4ef9-a426-98c67a387d82 2023-07-22T12:36:11,650 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xb81e3c9da16312bd: from storage DS-571338ac-e020-4cfa-a70a-55dc3041be92 node DatanodeRegistration(127.0.0.1:42097, datanodeUuid=de54bb37-b730-4ef9-a426-98c67a387d82, infoPort=35797, infoSecurePort=0, ipcPort=33727, storageInfo=lv=-57;cid=testClusterID;nsid=1660746884;c=1690029370861), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2023-07-22T12:36:11,650 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xb81e3c9da16312bd: Processing first storage report for DS-56489d71-fdda-4b68-8c83-781769addaca from datanode de54bb37-b730-4ef9-a426-98c67a387d82 2023-07-22T12:36:11,650 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xb81e3c9da16312bd: from storage DS-56489d71-fdda-4b68-8c83-781769addaca node DatanodeRegistration(127.0.0.1:42097, datanodeUuid=de54bb37-b730-4ef9-a426-98c67a387d82, infoPort=35797, infoSecurePort=0, ipcPort=33727, storageInfo=lv=-57;cid=testClusterID;nsid=1660746884;c=1690029370861), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-07-22T12:36:11,667 DEBUG [Listener at localhost/33727 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8 2023-07-22T12:36:11,667 INFO [Listener at localhost/33727 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-07-22T12:36:11,667 INFO [Listener at localhost/33727 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/test.cache.data in system properties and HBase conf 2023-07-22T12:36:11,667 INFO [Listener at localhost/33727 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/hadoop.tmp.dir in system properties and HBase conf 2023-07-22T12:36:11,667 INFO [Listener at localhost/33727 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/hadoop.log.dir in system properties and HBase conf 2023-07-22T12:36:11,667 INFO [Listener at localhost/33727 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/mapreduce.cluster.local.dir in system properties and HBase conf 2023-07-22T12:36:11,668 INFO [Listener at localhost/33727 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-07-22T12:36:11,668 INFO [Listener at localhost/33727 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-07-22T12:36:11,668 INFO [Listener at localhost/33727 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:36:11,669 INFO [Listener at localhost/33727 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:36:11,671 INFO [Listener at localhost/33727 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-07-22T12:36:11,671 INFO [Listener at localhost/33727 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-07-22T12:36:11,671 INFO [Listener at localhost/33727 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-07-22T12:36:11,671 INFO [Listener at localhost/33727 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-07-22T12:36:11,671 INFO [Listener at localhost/33727 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-07-22T12:36:11,671 INFO [Listener at localhost/33727 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-07-22T12:36:11,671 INFO [Listener at localhost/33727 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-07-22T12:36:11,672 INFO [Listener at localhost/33727 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/dfs.journalnode.edits.dir in system properties and HBase conf 2023-07-22T12:36:11,672 INFO [Listener at localhost/33727 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-07-22T12:36:11,672 INFO [Listener at localhost/33727 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/nfs.dump.dir in system properties and HBase conf 2023-07-22T12:36:11,672 INFO [Listener at localhost/33727 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/java.io.tmpdir in system properties and HBase conf 2023-07-22T12:36:11,672 INFO [Listener at localhost/33727 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/dfs.journalnode.edits.dir in system properties and HBase conf 2023-07-22T12:36:11,672 INFO [Listener at localhost/33727 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-07-22T12:36:11,672 INFO [Listener at localhost/33727 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-07-22T12:36:12,170 WARN [Thread-1335 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:36:12,175 WARN [Thread-1335 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-07-22T12:36:12,175 INFO [Thread-1335 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:36:12,183 INFO [Thread-1335 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_35875_jobhistory____.lcfosi/webapp 2023-07-22T12:36:12,236 INFO [Thread-1335 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-07-22T12:36:12,236 INFO [Thread-1335 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-07-22T12:36:12,236 INFO [Thread-1335 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-07-22T12:36:12,236 INFO [Thread-1335 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-07-22T12:36:12,243 INFO [Thread-1335 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:36:12,286 INFO [Thread-1335 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:36:12,380 INFO [Thread-1335 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-07-22T12:36:12,386 INFO [Thread-1335 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:35875 2023-07-22T12:36:13,659 WARN [Listener at jenkins-hbase3.apache.org/43165 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:36:13,667 WARN [Listener at jenkins-hbase3.apache.org/43165 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-07-22T12:36:13,668 INFO [Listener at jenkins-hbase3.apache.org/43165 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:36:13,676 INFO [Listener at jenkins-hbase3.apache.org/43165 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_46217_cluster____rb8ci5/webapp 2023-07-22T12:36:13,729 INFO [Listener at jenkins-hbase3.apache.org/43165 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-07-22T12:36:13,729 INFO [Listener at jenkins-hbase3.apache.org/43165 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-07-22T12:36:13,729 INFO [Listener at jenkins-hbase3.apache.org/43165 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-07-22T12:36:13,729 INFO [Listener at jenkins-hbase3.apache.org/43165 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-07-22T12:36:13,736 INFO [Listener at jenkins-hbase3.apache.org/43165 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:36:13,814 INFO [Listener at jenkins-hbase3.apache.org/43165 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:36:13,875 INFO [Listener at jenkins-hbase3.apache.org/43165 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:36:13,881 INFO [Listener at jenkins-hbase3.apache.org/43165 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:46217 2023-07-22T12:36:13,998 WARN [Listener at jenkins-hbase3.apache.org/44819 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:36:14,004 WARN [Listener at jenkins-hbase3.apache.org/44819 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-07-22T12:36:14,004 INFO [Listener at jenkins-hbase3.apache.org/44819 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:36:14,012 INFO [Listener at jenkins-hbase3.apache.org/44819 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_41325_node____gg2a0o/webapp 2023-07-22T12:36:14,065 INFO [Listener at jenkins-hbase3.apache.org/44819 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-07-22T12:36:14,065 INFO [Listener at jenkins-hbase3.apache.org/44819 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-07-22T12:36:14,065 INFO [Listener at jenkins-hbase3.apache.org/44819 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-07-22T12:36:14,065 INFO [Listener at jenkins-hbase3.apache.org/44819 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-07-22T12:36:14,072 INFO [Listener at jenkins-hbase3.apache.org/44819 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:36:14,087 INFO [Listener at jenkins-hbase3.apache.org/44819 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:36:14,155 INFO [Listener at jenkins-hbase3.apache.org/44819 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:36:14,164 INFO [Listener at jenkins-hbase3.apache.org/44819 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:41325 2023-07-22T12:36:14,251 WARN [Listener at jenkins-hbase3.apache.org/37741 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:36:14,257 WARN [Listener at jenkins-hbase3.apache.org/37741 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-07-22T12:36:14,257 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:36:14,264 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_34281_node____ffe2tl/webapp 2023-07-22T12:36:14,314 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-07-22T12:36:14,315 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-07-22T12:36:14,315 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-07-22T12:36:14,315 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-07-22T12:36:14,322 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:36:14,337 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:36:14,409 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:36:14,421 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:34281 2023-07-22T12:36:14,425 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-07-22T12:36:14,425 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1320e970 to 127.0.0.1:54609 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:36:14,432 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d07507b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:36:14,433 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:36:14,436 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:59436, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:36:14,439 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] impl.RestoreTablesClient(101): HBase table table1 does not exist. It will be created during restore process 2023-07-22T12:36:14,440 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38161/backupUT/backup_1690029359273/.backup.manifest 2023-07-22T12:36:14,443 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029359273/.backup.manifest 2023-07-22T12:36:14,444 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38161/backupUT/backup_1690029359273/.backup.manifest 2023-07-22T12:36:14,446 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029359273/.backup.manifest 2023-07-22T12:36:14,446 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] impl.RestoreTablesClient(148): Restoring 'test-1690029330567' to 'table1' from full backup image hdfs://localhost:38161/backupUT/backup_1690029359273/default/test-1690029330567 2023-07-22T12:36:14,455 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] util.RestoreTool(487): Creating target table 'table1' 2023-07-22T12:36:14,455 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:38161/backupUT/backup_1690029359273/default/test-1690029330567/archive/data/default/test-1690029330567/e2f9453920ffaa2bfd87a3743468e74e 2023-07-22T12:36:14,456 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:38161/backupUT/backup_1690029359273/default/test-1690029330567/archive/data/default/test-1690029330567/e2f9453920ffaa2bfd87a3743468e74e/f in region [hdfs://localhost:38161/backupUT/backup_1690029359273/default/test-1690029330567/archive/data/default/test-1690029330567/e2f9453920ffaa2bfd87a3743468e74e] 2023-07-22T12:36:14,460 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] util.RestoreTool(442): Trying to figure out region boundaries hfile=hdfs://localhost:38161/backupUT/backup_1690029359273/default/test-1690029330567/archive/data/default/test-1690029330567/e2f9453920ffaa2bfd87a3743468e74e/f/a663b1eb7c49421a9a8af7e19a9900bf first=row0 last=row98 2023-07-22T12:36:14,460 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a663b1eb7c49421a9a8af7e19a9900bf 2023-07-22T12:36:14,461 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:36:14,465 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:55332, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:36:14,466 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:36:14,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] procedure2.ProcedureExecutor(1033): Stored pid=47, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=table1 2023-07-22T12:36:14,471 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=47, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:36:14,471 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 47 2023-07-22T12:36:14,471 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=47, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:36:14,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=47 2023-07-22T12:36:14,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=47 2023-07-22T12:36:14,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=47 2023-07-22T12:36:14,883 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 97081e89385aac84fad3fbba8b4da366, NAME => 'table1,,1690029374466.97081e89385aac84fad3fbba8b4da366.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f 2023-07-22T12:36:15,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=47 2023-07-22T12:36:15,293 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(888): Instantiated table1,,1690029374466.97081e89385aac84fad3fbba8b4da366.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:36:15,293 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1668): Closing 97081e89385aac84fad3fbba8b4da366, disabling compactions & flushes 2023-07-22T12:36:15,293 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1690): Closing region table1,,1690029374466.97081e89385aac84fad3fbba8b4da366. 2023-07-22T12:36:15,293 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1690029374466.97081e89385aac84fad3fbba8b4da366. 2023-07-22T12:36:15,293 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on table1,,1690029374466.97081e89385aac84fad3fbba8b4da366. after waiting 0 ms 2023-07-22T12:36:15,293 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region table1,,1690029374466.97081e89385aac84fad3fbba8b4da366. 2023-07-22T12:36:15,293 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1909): Closed table1,,1690029374466.97081e89385aac84fad3fbba8b4da366. 2023-07-22T12:36:15,293 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1622): Region close journal for 97081e89385aac84fad3fbba8b4da366: 2023-07-22T12:36:15,294 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=47, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:36:15,295 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"table1,,1690029374466.97081e89385aac84fad3fbba8b4da366.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1690029375294"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029375294"}]},"ts":"1690029375294"} 2023-07-22T12:36:15,296 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:36:15,297 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=47, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:36:15,297 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029375297"}]},"ts":"1690029375297"} 2023-07-22T12:36:15,299 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLING in hbase:meta 2023-07-22T12:36:15,302 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=97081e89385aac84fad3fbba8b4da366, ASSIGN}] 2023-07-22T12:36:15,306 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=97081e89385aac84fad3fbba8b4da366, ASSIGN 2023-07-22T12:36:15,307 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=table1, region=97081e89385aac84fad3fbba8b4da366, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38147,1690029314593; forceNewPlan=false, retain=false 2023-07-22T12:36:15,460 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=48 updating hbase:meta row=97081e89385aac84fad3fbba8b4da366, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:36:15,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE; OpenRegionProcedure 97081e89385aac84fad3fbba8b4da366, server=jenkins-hbase3.apache.org,38147,1690029314593}] 2023-07-22T12:36:15,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=47 2023-07-22T12:36:15,627 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(130): Open table1,,1690029374466.97081e89385aac84fad3fbba8b4da366. 2023-07-22T12:36:15,628 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7254): Opening region: {ENCODED => 97081e89385aac84fad3fbba8b4da366, NAME => 'table1,,1690029374466.97081e89385aac84fad3fbba8b4da366.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:36:15,628 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:36:15,628 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 97081e89385aac84fad3fbba8b4da366 2023-07-22T12:36:15,628 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(888): Instantiated table1,,1690029374466.97081e89385aac84fad3fbba8b4da366.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:36:15,629 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7296): checking encryption for 97081e89385aac84fad3fbba8b4da366 2023-07-22T12:36:15,629 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7299): checking classloading for 97081e89385aac84fad3fbba8b4da366 2023-07-22T12:36:15,630 INFO [StoreOpener-97081e89385aac84fad3fbba8b4da366-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 97081e89385aac84fad3fbba8b4da366 2023-07-22T12:36:15,631 INFO [StoreOpener-97081e89385aac84fad3fbba8b4da366-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97081e89385aac84fad3fbba8b4da366 columnFamilyName f 2023-07-22T12:36:15,631 DEBUG [StoreOpener-97081e89385aac84fad3fbba8b4da366-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:36:15,632 INFO [StoreOpener-97081e89385aac84fad3fbba8b4da366-1 {}] regionserver.HStore(324): Store=97081e89385aac84fad3fbba8b4da366/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:36:15,632 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/table1/97081e89385aac84fad3fbba8b4da366 2023-07-22T12:36:15,633 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/table1/97081e89385aac84fad3fbba8b4da366 2023-07-22T12:36:15,635 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1076): writing seq id for 97081e89385aac84fad3fbba8b4da366 2023-07-22T12:36:15,637 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/table1/97081e89385aac84fad3fbba8b4da366/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:36:15,638 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): Opened 97081e89385aac84fad3fbba8b4da366; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10115895040, jitterRate=-0.0578838586807251}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:36:15,638 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(992): Region open journal for 97081e89385aac84fad3fbba8b4da366: 2023-07-22T12:36:15,639 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2552): Post open deploy tasks for table1,,1690029374466.97081e89385aac84fad3fbba8b4da366., pid=49, masterSystemTime=1690029375613 2023-07-22T12:36:15,640 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2579): Finished post open deploy task for table1,,1690029374466.97081e89385aac84fad3fbba8b4da366. 2023-07-22T12:36:15,640 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(158): Opened table1,,1690029374466.97081e89385aac84fad3fbba8b4da366. 2023-07-22T12:36:15,641 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=48 updating hbase:meta row=97081e89385aac84fad3fbba8b4da366, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:36:15,644 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=49, resume processing ppid=48 2023-07-22T12:36:15,644 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=49, ppid=48, state=SUCCESS; OpenRegionProcedure 97081e89385aac84fad3fbba8b4da366, server=jenkins-hbase3.apache.org,38147,1690029314593 in 180 msec 2023-07-22T12:36:15,646 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=48, resume processing ppid=47 2023-07-22T12:36:15,646 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=48, ppid=47, state=SUCCESS; TransitRegionStateProcedure table=table1, region=97081e89385aac84fad3fbba8b4da366, ASSIGN in 342 msec 2023-07-22T12:36:15,646 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=47, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:36:15,646 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029375646"}]},"ts":"1690029375646"} 2023-07-22T12:36:15,648 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLED in hbase:meta 2023-07-22T12:36:15,650 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=47, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:36:15,651 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=47, state=SUCCESS; CreateTableProcedure table=table1 in 1.1840 sec 2023-07-22T12:36:16,544 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'table1' 2023-07-22T12:36:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=47 2023-07-22T12:36:16,578 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: default:table1, procId: 47 completed 2023-07-22T12:36:16,588 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:38161/backupUT/backup_1690029359273/default/test-1690029330567/archive/data/default/test-1690029330567/e2f9453920ffaa2bfd87a3743468e74e from hbase tables test-1690029330567 to tables table1 2023-07-22T12:36:16,588 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.MapReduceRestoreJob(72): Restore test-1690029330567 into table1 2023-07-22T12:36:16,590 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:38685/backupUT/bulk_output-default-table1-1690029376588 from hdfs://localhost:38161/backupUT/backup_1690029359273/default/test-1690029330567/archive/data/default/test-1690029330567/e2f9453920ffaa2bfd87a3743468e74e 2023-07-22T12:36:16,590 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x47fbbd6b to 127.0.0.1:54609 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:36:16,596 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b3ba878, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:36:16,597 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:36:16,598 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:55338, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:36:16,600 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-07-22T12:36:16,600 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table table1 2023-07-22T12:36:16,605 DEBUG [hconnection-0x6bf889c8-shared-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:36:16,606 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:59442, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:36:16,608 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for table1: 2023-07-22T12:36:16,608 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-07-22T12:36:16,608 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to /user/jenkins/hbase-staging/partitions_7f73aa05-9a95-451e-9703-03ccb70cbe25 2023-07-22T12:36:16,617 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:36:16,618 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:36:16,618 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:36:16,619 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:36:17,582 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/hadoop-6971659716812695230.jar 2023-07-22T12:36:17,582 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:36:17,583 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:36:17,583 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:36:17,584 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:36:17,584 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:36:17,584 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:36:17,585 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:36:17,585 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:36:17,585 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:36:17,586 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:36:17,587 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:36:17,587 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:36:17,588 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:36:17,588 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:36:17,588 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:36:17,589 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:36:17,589 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:36:17,589 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:36:17,590 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:36:17,590 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:36:17,591 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:36:17,591 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:36:17,592 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:36:17,592 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:36:17,592 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:36:17,592 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:36:17,593 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:36:17,593 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:36:17,594 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: table1 2023-07-22T12:36:17,594 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] client.ConnectionImplementation(2095): Closing master protocol: MasterService 2023-07-22T12:36:17,594 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x47fbbd6b to 127.0.0.1:54609 2023-07-22T12:36:17,594 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:36:17,594 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-07-22T12:36:17,594 WARN [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-07-22T12:36:17,595 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:36:17,787 WARN [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-07-22T12:36:18,158 WARN [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-07-22T12:36:18,261 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region de72a5b968421854d430063920aa35db changed from -1.0 to 0.0, refreshing cache 2023-07-22T12:36:18,486 WARN [SchedulerEventDispatcher:Event Processor {}] capacity.LeafQueue(815): maximum-am-resource-percent is insufficient to start a single application in queue, it is likely set too low. skipping enforcement to allow at least one application to start 2023-07-22T12:36:18,486 WARN [SchedulerEventDispatcher:Event Processor {}] capacity.LeafQueue(851): maximum-am-resource-percent is insufficient to start a single application in queue for user, it is likely set too low. skipping enforcement to allow at least one application to start 2023-07-22T12:36:19,334 INFO [Socket Reader #1 for port 0 {}] ipc.Server$Connection(2001): Auth successful for appattempt_1690029373888_0001_000001 (auth:SIMPLE) 2023-07-22T12:36:21,442 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:36:21,939 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2023-07-22T12:36:21,939 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_table1 Metrics about Tables on a single HBase RegionServer 2023-07-22T12:36:23,767 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 88bacc909086c649c0d4264069cc9481 changed from -1.0 to 0.0, refreshing cache 2023-07-22T12:36:24,264 INFO [Socket Reader #1 for port 0 {}] ipc.Server$Connection(2001): Auth successful for appattempt_1690029373888_0001_000001 (auth:SIMPLE) 2023-07-22T12:36:26,534 INFO [Socket Reader #1 for port 0 {}] ipc.Server$Connection(2001): Auth successful for appattempt_1690029373888_0001_000001 (auth:SIMPLE) 2023-07-22T12:36:32,430 INFO [Socket Reader #1 for port 0 {}] ipc.Server$Connection(2001): Auth successful for appattempt_1690029373888_0001_000001 (auth:SIMPLE) 2023-07-22T12:36:35,607 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:49098, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:36:36,024 INFO [Socket Reader #1 for port 0 {}] ipc.Server$Connection(2001): Auth successful for appattempt_1690029373888_0001_000001 (auth:SIMPLE) 2023-07-22T12:36:36,042 WARN [ContainersLauncher #0 {}] nodemanager.DefaultContainerExecutor(305): Exit code from container container_1690029373888_0001_01_000003 is : 143 2023-07-22T12:36:37,932 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:38685/backupUT/bulk_output-default-table1-1690029376588 2023-07-22T12:36:37,933 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x489afd41 to 127.0.0.1:54609 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:36:37,942 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@444c06de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:36:37,943 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:36:37,947 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:49104, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:36:37,959 WARN [Listener at jenkins-hbase3.apache.org/37741 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:38685/backupUT/bulk_output-default-table1-1690029376588/_SUCCESS 2023-07-22T12:36:37,965 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:36:37,968 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:48780, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:36:37,991 DEBUG [LoadIncrementalHFiles-0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 838adb6a01b34859805d7b83696abbba 2023-07-22T12:36:37,991 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:38685/backupUT/bulk_output-default-table1-1690029376588/f/838adb6a01b34859805d7b83696abbba first=Optional[row0] last=Optional[row98] 2023-07-22T12:36:37,993 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=table1,,1690029374466.97081e89385aac84fad3fbba8b4da366., hostname=jenkins-hbase3.apache.org,38147,1690029314593, seqNum=2 for row with hfile group [{f,hdfs://localhost:38685/backupUT/bulk_output-default-table1-1690029376588/f/838adb6a01b34859805d7b83696abbba}] 2023-07-22T12:36:38,003 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:38685/backupUT/bulk_output-default-table1-1690029376588/f/838adb6a01b34859805d7b83696abbba for inclusion in 97081e89385aac84fad3fbba8b4da366/f 2023-07-22T12:36:38,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-07-22T12:36:38,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] regionserver.HStore(625): Region bounds: first= last= 2023-07-22T12:36:38,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 838adb6a01b34859805d7b83696abbba 2023-07-22T12:36:38,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] regionserver.HRegion(2520): Flush status journal for 97081e89385aac84fad3fbba8b4da366: 2023-07-22T12:36:38,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:38685/backupUT/bulk_output-default-table1-1690029376588/f/838adb6a01b34859805d7b83696abbba to hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/staging/jenkins__table1__caaju11l1tqtu6jbal63gtd3oh2oaroh0jcbag6hv3sqi71c9da99054rnabdudc/f/838adb6a01b34859805d7b83696abbba 2023-07-22T12:36:38,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/staging/jenkins__table1__caaju11l1tqtu6jbal63gtd3oh2oaroh0jcbag6hv3sqi71c9da99054rnabdudc/f/838adb6a01b34859805d7b83696abbba as hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/table1/97081e89385aac84fad3fbba8b4da366/f/b860636e0c1645b4a9c837ea862d6ee9_SeqId_4_ 2023-07-22T12:36:38,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6df2dc40 to 127.0.0.1:54609 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:36:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50468fd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:36:38,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:36:38,033 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:48794, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:36:38,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:36:38,039 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:49116, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:36:38,073 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] client.ConnectionImplementation(2095): Closing master protocol: MasterService 2023-07-22T12:36:38,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6df2dc40 to 127.0.0.1:54609 2023-07-22T12:36:38,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:36:38,074 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/staging/jenkins__table1__caaju11l1tqtu6jbal63gtd3oh2oaroh0jcbag6hv3sqi71c9da99054rnabdudc/f/838adb6a01b34859805d7b83696abbba into 97081e89385aac84fad3fbba8b4da366/f as hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/table1/97081e89385aac84fad3fbba8b4da366/f/b860636e0c1645b4a9c837ea862d6ee9_SeqId_4_ - updating store file list. 2023-07-22T12:36:38,083 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/table1/97081e89385aac84fad3fbba8b4da366/f/b860636e0c1645b4a9c837ea862d6ee9_SeqId_4_ into 97081e89385aac84fad3fbba8b4da366/f 2023-07-22T12:36:38,083 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/staging/jenkins__table1__caaju11l1tqtu6jbal63gtd3oh2oaroh0jcbag6hv3sqi71c9da99054rnabdudc/f/838adb6a01b34859805d7b83696abbba into 97081e89385aac84fad3fbba8b4da366/f (new location: hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/table1/97081e89385aac84fad3fbba8b4da366/f/b860636e0c1645b4a9c837ea862d6ee9_SeqId_4_) 2023-07-22T12:36:38,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/staging/jenkins__table1__caaju11l1tqtu6jbal63gtd3oh2oaroh0jcbag6hv3sqi71c9da99054rnabdudc/f/838adb6a01b34859805d7b83696abbba 2023-07-22T12:36:38,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0779170d to 127.0.0.1:54609 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:36:38,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e1d36c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:36:38,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:36:38,095 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:48804, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-07-22T12:36:38,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:36:38,103 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:49122, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-07-22T12:36:38,130 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] client.ConnectionImplementation(2095): Closing master protocol: MasterService 2023-07-22T12:36:38,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0779170d to 127.0.0.1:54609 2023-07-22T12:36:38,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38147 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:36:38,132 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] client.ConnectionImplementation(2095): Closing master protocol: MasterService 2023-07-22T12:36:38,134 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x489afd41 to 127.0.0.1:54609 2023-07-22T12:36:38,134 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:36:38,134 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-07-22T12:36:38,134 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-07-22T12:36:38,134 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] impl.RestoreTablesClient(233): Backup: backup_1690029359273 hdfs://localhost:38161/backupUT/backup_1690029359273/default/test-1690029330567/ 2023-07-22T12:36:38,134 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-07-22T12:36:38,147 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] client.HBaseAdmin$18(963): Started disable of table1 2023-07-22T12:36:38,147 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable table1 2023-07-22T12:36:38,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] procedure2.ProcedureExecutor(1033): Stored pid=50, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=table1 2023-07-22T12:36:38,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-07-22T12:36:38,154 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029398153"}]},"ts":"1690029398153"} 2023-07-22T12:36:38,155 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLING in hbase:meta 2023-07-22T12:36:38,157 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set table1 to state=DISABLING 2023-07-22T12:36:38,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=97081e89385aac84fad3fbba8b4da366, UNASSIGN}] 2023-07-22T12:36:38,161 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=51, ppid=50, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=97081e89385aac84fad3fbba8b4da366, UNASSIGN 2023-07-22T12:36:38,162 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=51 updating hbase:meta row=97081e89385aac84fad3fbba8b4da366, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:36:38,163 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE; CloseRegionProcedure 97081e89385aac84fad3fbba8b4da366, server=jenkins-hbase3.apache.org,38147,1690029314593}] 2023-07-22T12:36:38,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-07-22T12:36:38,315 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] handler.UnassignRegionHandler(119): Close 97081e89385aac84fad3fbba8b4da366 2023-07-22T12:36:38,315 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1668): Closing 97081e89385aac84fad3fbba8b4da366, disabling compactions & flushes 2023-07-22T12:36:38,315 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1690): Closing region table1,,1690029374466.97081e89385aac84fad3fbba8b4da366. 2023-07-22T12:36:38,315 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1690029374466.97081e89385aac84fad3fbba8b4da366. 2023-07-22T12:36:38,315 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1778): Acquired close lock on table1,,1690029374466.97081e89385aac84fad3fbba8b4da366. after waiting 0 ms 2023-07-22T12:36:38,315 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1788): Updates disabled for region table1,,1690029374466.97081e89385aac84fad3fbba8b4da366. 2023-07-22T12:36:38,318 DEBUG [StoreFileCloser-97081e89385aac84fad3fbba8b4da366-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: b860636e0c1645b4a9c837ea862d6ee9_SeqId_4_ 2023-07-22T12:36:38,327 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/table1/97081e89385aac84fad3fbba8b4da366/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-07-22T12:36:38,328 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:36:38,329 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1909): Closed table1,,1690029374466.97081e89385aac84fad3fbba8b4da366. 2023-07-22T12:36:38,329 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1622): Region close journal for 97081e89385aac84fad3fbba8b4da366: 2023-07-22T12:36:38,331 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] handler.UnassignRegionHandler(163): Closed 97081e89385aac84fad3fbba8b4da366 2023-07-22T12:36:38,331 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=51 updating hbase:meta row=97081e89385aac84fad3fbba8b4da366, regionState=CLOSED 2023-07-22T12:36:38,334 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=52, resume processing ppid=51 2023-07-22T12:36:38,334 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=52, ppid=51, state=SUCCESS; CloseRegionProcedure 97081e89385aac84fad3fbba8b4da366, server=jenkins-hbase3.apache.org,38147,1690029314593 in 170 msec 2023-07-22T12:36:38,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=51, resume processing ppid=50 2023-07-22T12:36:38,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=51, ppid=50, state=SUCCESS; TransitRegionStateProcedure table=table1, region=97081e89385aac84fad3fbba8b4da366, UNASSIGN in 177 msec 2023-07-22T12:36:38,337 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029398337"}]},"ts":"1690029398337"} 2023-07-22T12:36:38,339 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLED in hbase:meta 2023-07-22T12:36:38,340 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set table1 to state=DISABLED 2023-07-22T12:36:38,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=50, state=SUCCESS; DisableTableProcedure table=table1 in 194 msec 2023-07-22T12:36:38,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-07-22T12:36:38,455 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] client.HBaseAdmin$TableFuture(3590): Operation: DISABLE, Table Name: default:table1, procId: 50 completed 2023-07-22T12:36:38,456 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.HMaster$5(2447): Client=jenkins//172.31.12.81 delete table1 2023-07-22T12:36:38,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] procedure2.ProcedureExecutor(1033): Stored pid=53, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=table1 2023-07-22T12:36:38,461 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=53, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-07-22T12:36:38,461 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=53, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=table1 2023-07-22T12:36:38,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=53 2023-07-22T12:36:38,464 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/table1/97081e89385aac84fad3fbba8b4da366 2023-07-22T12:36:38,467 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/table1/97081e89385aac84fad3fbba8b4da366/f, FileablePath, hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/table1/97081e89385aac84fad3fbba8b4da366/recovered.edits] 2023-07-22T12:36:38,479 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/table1/97081e89385aac84fad3fbba8b4da366/f/b860636e0c1645b4a9c837ea862d6ee9_SeqId_4_ to hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/archive/data/default/table1/97081e89385aac84fad3fbba8b4da366/f/b860636e0c1645b4a9c837ea862d6ee9_SeqId_4_ 2023-07-22T12:36:38,483 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/table1/97081e89385aac84fad3fbba8b4da366/recovered.edits/6.seqid to hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/archive/data/default/table1/97081e89385aac84fad3fbba8b4da366/recovered.edits/6.seqid 2023-07-22T12:36:38,483 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/table1/97081e89385aac84fad3fbba8b4da366 2023-07-22T12:36:38,484 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(293): Archived table1 regions 2023-07-22T12:36:38,486 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=53, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=table1 2023-07-22T12:36:38,492 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of table1 from hbase:meta 2023-07-22T12:36:38,494 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(387): Removing 'table1' descriptor. 2023-07-22T12:36:38,496 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=53, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=table1 2023-07-22T12:36:38,496 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(377): Removing 'table1' from region states. 2023-07-22T12:36:38,496 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1,,1690029374466.97081e89385aac84fad3fbba8b4da366.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1690029398496"}]},"ts":"9223372036854775807"} 2023-07-22T12:36:38,499 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-07-22T12:36:38,499 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 97081e89385aac84fad3fbba8b4da366, NAME => 'table1,,1690029374466.97081e89385aac84fad3fbba8b4da366.', STARTKEY => '', ENDKEY => ''}] 2023-07-22T12:36:38,500 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(381): Marking 'table1' as deleted. 2023-07-22T12:36:38,500 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1690029398500"}]},"ts":"9223372036854775807"} 2023-07-22T12:36:38,501 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1678): Deleted table table1 state from META 2023-07-22T12:36:38,503 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(127): Finished pid=53, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-07-22T12:36:38,504 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=53, state=SUCCESS; DeleteTableProcedure table=table1 in 47 msec 2023-07-22T12:36:38,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46137 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=53 2023-07-22T12:36:38,564 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] client.HBaseAdmin$TableFuture(3590): Operation: DELETE, Table Name: default:table1, procId: 53 completed 2023-07-22T12:36:38,635 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] hbase.ResourceChecker(175): after: backup.TestRemoteRestore#testFullRestoreRemoteWithAlternateRestoreOutputDir Thread=1302 (was 831) Potentially hanging thread: 2005397389@qtp-379735091-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:43003 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: PacketResponder: BP-1697032271-172.31.12.81-1690029308490:blk_1073741876_1052, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 42 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 25 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 7 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Client (2082644736) connection to localhost/127.0.0.1:40741 from jenkins java.lang.Object.wait(Native Method) org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1053) org.apache.hadoop.ipc.Client$Connection.run(Client.java:1097) Potentially hanging thread: IPC Server handler 31 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 1 on default port 44903 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 39 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O server boss #68 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.jboss.netty.channel.socket.nio.NioServerBoss.select(NioServerBoss.java:163) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.NioServerBoss.run(NioServerBoss.java:42) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 5 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 1 on default port 33727 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-362-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 43 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 38 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 4 on default port 33727 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_344205816_16 at /127.0.0.1:44406 [Waiting for operation #8] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: pool-349-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: 1116984019@qtp-277382704-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:46217 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: IPC Server handler 24 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 20 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-60 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 1 on default port 44819 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 19 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 14 on default port 46487 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 7 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 13 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Client (2082644736) connection to jenkins-hbase3.apache.org/172.31.12.81:34481 from jenkins java.lang.Object.wait(Native Method) org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1053) org.apache.hadoop.ipc.Client$Connection.run(Client.java:1097) Potentially hanging thread: Listener at jenkins-hbase3.apache.org/37741 java.lang.Thread.dumpThreads(Native Method) java.lang.Thread.getAllStackTraces(Thread.java:1615) org.apache.hadoop.hbase.ResourceCheckerJUnitListener$ThreadResourceAnalyzer.getVal(ResourceCheckerJUnitListener.java:49) org.apache.hadoop.hbase.ResourceChecker.fill(ResourceChecker.java:110) org.apache.hadoop.hbase.ResourceChecker.fillEndings(ResourceChecker.java:104) org.apache.hadoop.hbase.ResourceChecker.end(ResourceChecker.java:206) org.apache.hadoop.hbase.ResourceCheckerJUnitListener.end(ResourceCheckerJUnitListener.java:165) org.apache.hadoop.hbase.ResourceCheckerJUnitListener.testFinished(ResourceCheckerJUnitListener.java:185) org.junit.runner.notification.SynchronizedRunListener.testFinished(SynchronizedRunListener.java:87) org.junit.runner.notification.RunNotifier$9.notifyListener(RunNotifier.java:225) org.junit.runner.notification.RunNotifier$SafeNotifier.run(RunNotifier.java:72) org.junit.runner.notification.RunNotifier.fireTestFinished(RunNotifier.java:222) org.junit.internal.runners.model.EachTestNotifier.fireTestFinished(EachTestNotifier.java:38) org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:372) org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) java.util.concurrent.FutureTask.run(FutureTask.java:266) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: 1478516907@qtp-1957524769-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: New I/O worker #61 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 1 on default port 40741 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@2e0d55e0 java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/cluster_c16b38c6-f9ae-d092-5fc7-64996592badb/dfs/data/data4) java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:627) Potentially hanging thread: IPC Server handler 3 on default port 41929 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.hdfs.server.datanode.DataXceiverServer@64bc45b6 sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:421) sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:249) sun.nio.ch.ServerSocketAdaptor.accept(ServerSocketAdaptor.java:113) org.apache.hadoop.hdfs.net.TcpPeerServer.accept(TcpPeerServer.java:85) org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.run(DataXceiverServer.java:145) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 2 on default port 44819 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 4 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #54 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #46 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-972608796_16 at /127.0.0.1:43514 [Receiving block BP-1697032271-172.31.12.81-1690029308490:blk_1073741876_1052] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 23 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 7 on default port 46487 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@4ac0fb50 java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 9 on default port 46487 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/cluster_c16b38c6-f9ae-d092-5fc7-64996592badb/dfs/data/data5/current/BP-2039514963-172.31.12.81-1690029370861 java.lang.Thread.sleep(Native Method) org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:205) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: 315779639@qtp-1168490686-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: IPC Server handler 27 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #57 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 47 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-62 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 47 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 44 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 49 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 46 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 4 on default port 46487 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:54609@0x1320e970 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:328) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$494/990948707.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ForkJoinPool-2-worker-6 sun.misc.Unsafe.park(Native Method) java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1824) java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1693) java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) Potentially hanging thread: org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor@25ea2f3d java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor.run(LeaseManager.java:533) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1494279943_1 at /127.0.0.1:52374 [Waiting for operation #2] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 3 on default port 44819 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 6 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 8 on default port 39043 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-67 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 21 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #47 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 18 on default port 46487 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 8 on default port 43467 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 29 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 764978393@qtp-1168490686-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:34053 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: Thread-2086 java.io.FileInputStream.readBytes(Native Method) java.io.FileInputStream.read(FileInputStream.java:255) java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.io.BufferedInputStream.read(BufferedInputStream.java:345) sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:284) sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:326) sun.nio.cs.StreamDecoder.read(StreamDecoder.java:178) java.io.InputStreamReader.read(InputStreamReader.java:184) java.io.BufferedReader.fill(BufferedReader.java:161) java.io.BufferedReader.readLine(BufferedReader.java:324) java.io.BufferedReader.readLine(BufferedReader.java:389) org.apache.hadoop.util.Shell$1.run(Shell.java:955) Potentially hanging thread: IPC Server handler 0 on default port 41929 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-64 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 13 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: region-location-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server idle connection scanner for port 40741 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 7 on default port 39043 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 41 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 43 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 43165 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 4 on default port 40741 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 37741 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-55 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 32 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 41929 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 3 on default port 33727 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.hdfs.server.namenode.FSNamesystem$LazyPersistFileScrubber@4fded2c7 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.server.namenode.FSNamesystem$LazyPersistFileScrubber.run(FSNamesystem.java:3975) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 28 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 5 on default port 39043 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-65 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 9 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 1120314206@qtp-535527795-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:40717 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: IPC Server handler 49 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 12 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 20 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #65 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/cluster_c16b38c6-f9ae-d092-5fc7-64996592badb/dfs/data/data1/current/BP-2039514963-172.31.12.81-1690029370861 java.lang.Thread.sleep(Native Method) org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:205) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 18 on default port 43467 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 41 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 34757 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 14 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #63 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 31 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-48 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 2 on default port 40741 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 23 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 6 on default port 39043 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 1 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 38451 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: org.apache.hadoop.hdfs.server.datanode.DataXceiverServer@56b4776f sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:421) sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:249) sun.nio.ch.ServerSocketAdaptor.accept(ServerSocketAdaptor.java:113) org.apache.hadoop.hdfs.net.TcpPeerServer.accept(TcpPeerServer.java:85) org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.run(DataXceiverServer.java:145) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 22 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: CacheReplicationMonitor(1776410686) sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) org.apache.hadoop.hdfs.server.blockmanagement.CacheReplicationMonitor.run(CacheReplicationMonitor.java:181) Potentially hanging thread: IPC Server handler 14 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 10 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 33727 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 35 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:54609@0x1320e970-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: hconnection-0x552edb61-shared-pool-17 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 4 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@21d8a8d java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: pool-356-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 11 on default port 46487 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 8 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 32 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: HFileArchiver-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 40 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@534737f3 java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 17 on default port 43467 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Thread-1378 java.lang.Thread.sleep(Native Method) org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:143) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 17 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-45 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 3 on default port 43467 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 1408872854@qtp-949299009-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/cluster_c16b38c6-f9ae-d092-5fc7-64996592badb/dfs/data/data4/current/BP-2039514963-172.31.12.81-1690029370861 java.lang.Thread.sleep(Native Method) org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:205) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeEditLogRoller@2dbd4986 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeEditLogRoller.run(FSNamesystem.java:3883) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 24 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 1 on default port 39043 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 5 on default port 43467 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 1 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 40 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #40 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 2 on default port 44903 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 8 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 46487 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #38 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: 527832553@qtp-1952393397-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: New I/O worker #45 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: BP-2039514963-172.31.12.81-1690029370861 heartbeating to localhost/127.0.0.1:40741 java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.IncrementalBlockReportManager.waitTillNextIBR(IncrementalBlockReportManager.java:158) org.apache.hadoop.hdfs.server.datanode.BPServiceActor.offerService(BPServiceActor.java:715) org.apache.hadoop.hdfs.server.datanode.BPServiceActor.run(BPServiceActor.java:851) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 33 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #67 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: nioEventLoopGroup-10-1 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:803) io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457) io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 38 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #42 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 37 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 39 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 30 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 37741 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 44337 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: ContainersLauncher #0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ForkJoinPool-2-worker-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1824) java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1693) java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) Potentially hanging thread: IPC Server handler 37 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 48 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:54609@0x1320e970-SendThread(127.0.0.1:54609) sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:345) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1223) Potentially hanging thread: Timer-39 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server idle connection scanner for port 44903 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor@4d628816 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor.run(HeartbeatManager.java:536) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@3ddbf542 java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #43 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #35 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: LeaseRenewer:jenkins@localhost:40741 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:412) org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$600(LeaseRenewer.java:76) org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:308) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 9 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@7f5c811[State = -1, empty queue] sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DeletionService #1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 4 on default port 44819 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 4 on default port 37741 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 32 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 41 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 530914551@qtp-379735091-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: New I/O worker #39 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 44 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 27 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 17 on default port 46487 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 43467 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 35 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 11 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-972608796_16 at /127.0.0.1:43524 [Receiving block BP-1697032271-172.31.12.81-1690029308490:blk_1073741877_1053] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 16 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O server boss #51 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.jboss.netty.channel.socket.nio.NioServerBoss.select(NioServerBoss.java:163) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.NioServerBoss.run(NioServerBoss.java:42) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: pool-398-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-47 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 12 on default port 43467 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 46 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #52 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 1 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 14 on default port 43467 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 40 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Thread[Thread-1361,5,FailOnTimeoutGroup] java.lang.Thread.sleep(Native Method) org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:694) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: 1806577681@qtp-1673629086-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: IPC Server handler 42 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: hconnection-0x552edb61-shared-pool-16 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-41 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server idle connection scanner for port 44819 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 21 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 40741 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: hconnection-0x552edb61-shared-pool-15 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 15 on default port 43467 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 3 on default port 40741 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-53 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 0 on default port 44337 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 43467 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/cluster_c16b38c6-f9ae-d092-5fc7-64996592badb/dfs/data/data2/current/BP-2039514963-172.31.12.81-1690029370861 java.lang.Thread.sleep(Native Method) org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:205) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 21 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 26 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-50 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: New I/O worker #41 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 3 on default port 46487 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 5 on default port 46487 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 26 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #49 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #64 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 25 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 11 on default port 43467 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 33727 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: pool-358-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 47 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 8 on default port 46487 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 11 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.hdfs.server.datanode.DataXceiverServer@46d42aab sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:421) sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:249) sun.nio.ch.ServerSocketAdaptor.accept(ServerSocketAdaptor.java:113) org.apache.hadoop.hdfs.net.TcpPeerServer.accept(TcpPeerServer.java:85) org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.run(DataXceiverServer.java:145) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/cluster_c16b38c6-f9ae-d092-5fc7-64996592badb/dfs/data/data1) java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:627) Potentially hanging thread: New I/O worker #53 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: MutableQuantiles-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@a8b971f java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: BP-2039514963-172.31.12.81-1690029370861 heartbeating to localhost/127.0.0.1:40741 java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.IncrementalBlockReportManager.waitTillNextIBR(IncrementalBlockReportManager.java:158) org.apache.hadoop.hdfs.server.datanode.BPServiceActor.offerService(BPServiceActor.java:715) org.apache.hadoop.hdfs.server.datanode.BPServiceActor.run(BPServiceActor.java:851) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x552edb61-shared-pool-13 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 1 on default port 41929 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 15 on default port 46487 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #37 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 3 on default port 37741 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-42 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 18 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-56 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 10 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 17 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 19 on default port 46487 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 30 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 16 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 4 on default port 43467 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 30 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 20 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 13 on default port 43467 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 39 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/cluster_c16b38c6-f9ae-d092-5fc7-64996592badb/dfs/data/data3/current/BP-2039514963-172.31.12.81-1690029370861 java.lang.Thread.sleep(Native Method) org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:205) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@2712995d java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 1 on default port 43467 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Thread[Thread-1371,5,FailOnTimeoutGroup] java.lang.Thread.sleep(Native Method) org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:694) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 18 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/cluster_c16b38c6-f9ae-d092-5fc7-64996592badb/dfs/data/data6/current/BP-2039514963-172.31.12.81-1690029370861 java.lang.Thread.sleep(Native Method) org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:205) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ContainersLauncher #0 java.io.FileInputStream.readBytes(Native Method) java.io.FileInputStream.read(FileInputStream.java:255) java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.io.BufferedInputStream.read(BufferedInputStream.java:345) sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:284) sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:326) sun.nio.cs.StreamDecoder.read(StreamDecoder.java:178) java.io.InputStreamReader.read(InputStreamReader.java:184) java.io.BufferedReader.fill(BufferedReader.java:161) java.io.BufferedReader.read1(BufferedReader.java:212) java.io.BufferedReader.read(BufferedReader.java:286) org.apache.hadoop.util.Shell$ShellCommandExecutor.parseExecResult(Shell.java:1229) org.apache.hadoop.util.Shell.runCommand(Shell.java:984) org.apache.hadoop.util.Shell.run(Shell.java:884) org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1216) org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor.launchContainer(DefaultContainerExecutor.java:294) org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.launchContainer(ContainerLaunch.java:447) org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.call(ContainerLaunch.java:298) org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.call(ContainerLaunch.java:99) java.util.concurrent.FutureTask.run(FutureTask.java:266) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 15 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 3 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 5 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@1e9ffc7c java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 34 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 3 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 1907199326@qtp-1673629086-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:35875 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: IPC Server handler 24 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-372-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #58 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 44 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 19 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: DeletionService #0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 0 on default port 39043 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #62 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 0 on default port 44903 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Container metrics unregistration java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 6 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-49 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 45 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 33 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 41929 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 5 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 12 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Client (2082644736) connection to localhost/127.0.0.1:40741 from jenkins java.lang.Object.wait(Native Method) org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1053) org.apache.hadoop.ipc.Client$Connection.run(Client.java:1097) Potentially hanging thread: PacketResponder: BP-1697032271-172.31.12.81-1690029308490:blk_1073741877_1053, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 17 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-400-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@29901987[State = -1, empty queue] sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 0 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 10 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-351-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 43 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #50 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: pool-368-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 36 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 16 on default port 46487 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 1 on default port 46487 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 19 on default port 43467 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #66 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 13 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 10 on default port 43467 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 49 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #44 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 25 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 31 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 7 on default port 43467 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: hconnection-0x552edb61-shared-pool-18 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 37 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 9 on default port 39043 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/cluster_c16b38c6-f9ae-d092-5fc7-64996592badb/dfs/data/data5) java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:627) Potentially hanging thread: Timer-51 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: Timer-66 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: New I/O worker #59 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 13 on default port 46487 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 39043 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 1062746806@qtp-949299009-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:41673 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: IPC Server handler 15 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-395-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #60 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: pool-346-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 33 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 34 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-58 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: pool-344-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 6 on default port 43467 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-352-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 3 on default port 39043 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 43165 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.hdfs.server.blockmanagement.PendingReplicationBlocks$PendingReplicationMonitor@34435fc3 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.server.blockmanagement.PendingReplicationBlocks$PendingReplicationMonitor.run(PendingReplicationBlocks.java:244) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 19 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #36 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DeletionService #3 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-40 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 28 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: nioEventLoopGroup-12-1 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:803) io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457) io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-57 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: 819371658@qtp-277382704-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@4d9261eb[State = -1, empty queue] sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server idle connection scanner for port 34205 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server idle connection scanner for port 37741 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 42 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 45 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-63 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 0 on default port 43467 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #55 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 0 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 1967614935@qtp-535527795-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: IPC Server handler 7 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #56 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: nioEventLoopGroup-14-1 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:803) io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457) io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 10 on default port 46487 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: hconnection-0x552edb61-shared-pool-14 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: 675567351@qtp-1957524769-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:41325 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: IPC Server handler 8 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 9 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 48 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 14 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 29 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/cluster_c16b38c6-f9ae-d092-5fc7-64996592badb/dfs/data/data6) java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:627) Potentially hanging thread: IPC Server handler 46 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1494279943_1 at /127.0.0.1:51528 [Waiting for operation #2] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-52 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: New I/O worker #48 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-59 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 38 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: DeletionService #0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 0 on default port 46487 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Thread[Thread-1336,5,FailOnTimeoutGroup] java.lang.Thread.sleep(Native Method) org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:694) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 26 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-43 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 36 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 4 on default port 44903 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-44 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 9 on default port 43467 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 15 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 22 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 23 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 33727 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-46 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 16 on default port 43467 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 12 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 6 on default port 46487 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 29 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 4 on default port 41929 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 27 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-366-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 34 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: ApplicationMasterLauncher #0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: region-location-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 48 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-54 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 2 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 12 on default port 46487 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 3 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 11 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: DeletionService #2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 16 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-61 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 4 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 44819 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 28 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 3 on default port 44903 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 4 on default port 39043 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: BP-2039514963-172.31.12.81-1690029370861 heartbeating to localhost/127.0.0.1:40741 java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.IncrementalBlockReportManager.waitTillNextIBR(IncrementalBlockReportManager.java:158) org.apache.hadoop.hdfs.server.datanode.BPServiceActor.offerService(BPServiceActor.java:715) org.apache.hadoop.hdfs.server.datanode.BPServiceActor.run(BPServiceActor.java:851) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server idle connection scanner for port 39043 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/cluster_c16b38c6-f9ae-d092-5fc7-64996592badb/dfs/data/data3) java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:627) Potentially hanging thread: VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5ae44275-6dbb-56a9-5dff-3be2ced88cf8/cluster_c16b38c6-f9ae-d092-5fc7-64996592badb/dfs/data/data2) java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:627) Potentially hanging thread: IPC Server handler 22 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 1 on default port 37741 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 46487 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 36 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeResourceMonitor@3977b79 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeResourceMonitor.run(FSNamesystem.java:3841) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 35 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 45 on default port 34757 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 1998389501@qtp-1952393397-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:34281 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: IPC Server handler 18 on default port 38451 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-396-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 6 on default port 34205 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) - Thread LEAK? -, OpenFileDescriptor=1256 (was 903) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=219 (was 90) - SystemLoadAverage LEAK? -, ProcessCount=172 (was 169) - ProcessCount LEAK? -, AvailableMemoryMB=16290 (was 16457) 2023-07-22T12:36:38,639 WARN [Listener at jenkins-hbase3.apache.org/37741 {}] hbase.ResourceChecker(130): Thread=1302 is superior to 500 2023-07-22T12:36:38,639 WARN [Listener at jenkins-hbase3.apache.org/37741 {}] hbase.ResourceChecker(130): OpenFileDescriptor=1256 is superior to 1024 2023-07-22T12:36:38,648 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-07-22T12:36:38,648 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x431071ca to 127.0.0.1:54609 2023-07-22T12:36:38,648 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:36:38,655 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-07-22T12:36:38,655 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] util.JVMClusterUtil(257): Found active master hash=2073965613, stopped=false 2023-07-22T12:36:38,656 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] master.ServerManager(910): Cluster shutdown requested of master=jenkins-hbase3.apache.org,46439,1690029321204 2023-07-22T12:36:38,660 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39575-0x100006222d70005, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/2/running 2023-07-22T12:36:38,660 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39575-0x100006222d70005, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-07-22T12:36:38,660 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/2/running 2023-07-22T12:36:38,660 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-07-22T12:36:38,661 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39575-0x100006222d70005, quorum=127.0.0.1:54609, baseZNode=/2 Set watcher on znode that does not yet exist, /2/running 2023-07-22T12:36:38,661 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Set watcher on znode that does not yet exist, /2/running 2023-07-22T12:36:38,661 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] procedure2.ProcedureExecutor(631): Stopping 2023-07-22T12:36:38,662 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:36:38,662 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] regionserver.HRegionServer(2513): ***** STOPPING region server 'jenkins-hbase3.apache.org,39575,1690029321362' ***** 2023-07-22T12:36:38,662 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] regionserver.HRegionServer(2527): STOPPED: Shutdown requested 2023-07-22T12:36:38,663 INFO [RS:0;jenkins-hbase3:39575 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-07-22T12:36:38,691 INFO [RS:0;jenkins-hbase3:39575 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@13f002cd{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-07-22T12:36:38,696 INFO [RS:0;jenkins-hbase3:39575 {}] server.AbstractConnector(383): Stopped ServerConnector@2f51b5c0{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-07-22T12:36:38,696 INFO [RS:0;jenkins-hbase3:39575 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-07-22T12:36:38,698 INFO [RS:0;jenkins-hbase3:39575 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@38041cf0{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-07-22T12:36:38,700 INFO [RS:0;jenkins-hbase3:39575 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@44bed0d6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/hadoop.log.dir/,STOPPED} 2023-07-22T12:36:38,702 INFO [RS:0;jenkins-hbase3:39575 {}] regionserver.HeapMemoryManager(220): Stopping 2023-07-22T12:36:38,702 INFO [RS:0;jenkins-hbase3:39575 {}] flush.RegionServerFlushTableProcedureManager(117): Stopping region server flush procedure manager gracefully. 2023-07-22T12:36:38,702 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-07-22T12:36:38,702 INFO [RS:0;jenkins-hbase3:39575 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-07-22T12:36:38,702 INFO [RS:0;jenkins-hbase3:39575 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-07-22T12:36:38,703 INFO [RS:0;jenkins-hbase3:39575 {}] regionserver.HRegionServer(3524): Received CLOSE for 88bacc909086c649c0d4264069cc9481 2023-07-22T12:36:38,703 INFO [RS:0;jenkins-hbase3:39575 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,39575,1690029321362 2023-07-22T12:36:38,703 DEBUG [RS:0;jenkins-hbase3:39575 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:36:38,705 INFO [RS:0;jenkins-hbase3:39575 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-07-22T12:36:38,705 INFO [RS:0;jenkins-hbase3:39575 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-07-22T12:36:38,705 INFO [RS:0;jenkins-hbase3:39575 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-07-22T12:36:38,705 INFO [RS:0;jenkins-hbase3:39575 {}] regionserver.HRegionServer(3524): Received CLOSE for 1588230740 2023-07-22T12:36:38,706 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 88bacc909086c649c0d4264069cc9481, disabling compactions & flushes 2023-07-22T12:36:38,706 INFO [RS:0;jenkins-hbase3:39575 {}] regionserver.HRegionServer(1579): Waiting on 2 regions to close 2023-07-22T12:36:38,706 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1690029323767.88bacc909086c649c0d4264069cc9481. 2023-07-22T12:36:38,706 DEBUG [RS:0;jenkins-hbase3:39575 {}] regionserver.HRegionServer(1583): Online Regions={1588230740=hbase:meta,,1.1588230740, 88bacc909086c649c0d4264069cc9481=hbase:namespace,,1690029323767.88bacc909086c649c0d4264069cc9481.} 2023-07-22T12:36:38,708 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1690029323767.88bacc909086c649c0d4264069cc9481. 2023-07-22T12:36:38,708 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-07-22T12:36:38,708 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1690029323767.88bacc909086c649c0d4264069cc9481. after waiting 0 ms 2023-07-22T12:36:38,708 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1690029323767.88bacc909086c649c0d4264069cc9481. 2023-07-22T12:36:38,708 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-07-22T12:36:38,708 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-07-22T12:36:38,708 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 88bacc909086c649c0d4264069cc9481 1/1 column families, dataSize=78 B heapSize=488 B 2023-07-22T12:36:38,708 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-07-22T12:36:38,708 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-07-22T12:36:38,711 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=1.26 KB heapSize=2.89 KB 2023-07-22T12:36:38,712 DEBUG [RS:0;jenkins-hbase3:39575 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 88bacc909086c649c0d4264069cc9481 2023-07-22T12:36:38,737 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/data/hbase/namespace/88bacc909086c649c0d4264069cc9481/.tmp/info/360b83b59b3a415db52aa29d9abbdce5 is 45, key is default/info:d/1690029325085/Put/seqid=0 2023-07-22T12:36:38,742 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/data/hbase/meta/1588230740/.tmp/info/9bcb0fd7964e4f0bb3b288296e47f9aa is 143, key is hbase:namespace,,1690029323767.88bacc909086c649c0d4264069cc9481./info:regioninfo/1690029324975/Put/seqid=0 2023-07-22T12:36:38,755 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-07-22T12:36:38,912 DEBUG [RS:0;jenkins-hbase3:39575 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 88bacc909086c649c0d4264069cc9481 2023-07-22T12:36:38,978 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-07-22T12:36:38,978 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-07-22T12:36:39,112 DEBUG [RS:0;jenkins-hbase3:39575 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 88bacc909086c649c0d4264069cc9481 2023-07-22T12:36:39,147 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.17 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/data/hbase/meta/1588230740/.tmp/info/9bcb0fd7964e4f0bb3b288296e47f9aa 2023-07-22T12:36:39,159 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/data/hbase/namespace/88bacc909086c649c0d4264069cc9481/.tmp/info/360b83b59b3a415db52aa29d9abbdce5 2023-07-22T12:36:39,168 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 360b83b59b3a415db52aa29d9abbdce5 2023-07-22T12:36:39,168 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 9bcb0fd7964e4f0bb3b288296e47f9aa 2023-07-22T12:36:39,169 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/data/hbase/namespace/88bacc909086c649c0d4264069cc9481/.tmp/info/360b83b59b3a415db52aa29d9abbdce5 as hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/data/hbase/namespace/88bacc909086c649c0d4264069cc9481/info/360b83b59b3a415db52aa29d9abbdce5 2023-07-22T12:36:39,175 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/data/hbase/namespace/88bacc909086c649c0d4264069cc9481/info/360b83b59b3a415db52aa29d9abbdce5, entries=2, sequenceid=6, filesize=4.9 K 2023-07-22T12:36:39,176 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 88bacc909086c649c0d4264069cc9481 in 468ms, sequenceid=6, compaction requested=false 2023-07-22T12:36:39,181 DEBUG [StoreFileCloser-88bacc909086c649c0d4264069cc9481-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 360b83b59b3a415db52aa29d9abbdce5 2023-07-22T12:36:39,189 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38161/tmp/wal/data/hbase/namespace/88bacc909086c649c0d4264069cc9481/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2023-07-22T12:36:39,190 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:36:39,192 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1690029323767.88bacc909086c649c0d4264069cc9481. 2023-07-22T12:36:39,192 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 88bacc909086c649c0d4264069cc9481: 2023-07-22T12:36:39,192 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/data/hbase/meta/1588230740/.tmp/table/6df1dc4958d04bf09fc29f60ce3010ba is 51, key is hbase:namespace/table:state/1690029324986/Put/seqid=0 2023-07-22T12:36:39,192 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1690029323767.88bacc909086c649c0d4264069cc9481. 2023-07-22T12:36:39,199 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=94 B at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/data/hbase/meta/1588230740/.tmp/table/6df1dc4958d04bf09fc29f60ce3010ba 2023-07-22T12:36:39,206 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6df1dc4958d04bf09fc29f60ce3010ba 2023-07-22T12:36:39,207 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/data/hbase/meta/1588230740/.tmp/info/9bcb0fd7964e4f0bb3b288296e47f9aa as hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/data/hbase/meta/1588230740/info/9bcb0fd7964e4f0bb3b288296e47f9aa 2023-07-22T12:36:39,212 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/data/hbase/meta/1588230740/info/9bcb0fd7964e4f0bb3b288296e47f9aa, entries=10, sequenceid=9, filesize=6.4 K 2023-07-22T12:36:39,213 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/data/hbase/meta/1588230740/.tmp/table/6df1dc4958d04bf09fc29f60ce3010ba as hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/data/hbase/meta/1588230740/table/6df1dc4958d04bf09fc29f60ce3010ba 2023-07-22T12:36:39,221 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/data/hbase/meta/1588230740/table/6df1dc4958d04bf09fc29f60ce3010ba, entries=2, sequenceid=9, filesize=5.1 K 2023-07-22T12:36:39,222 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~1.26 KB/1290, heapSize ~2.61 KB/2672, currentSize=0 B/0 for 1588230740 in 514ms, sequenceid=9, compaction requested=false 2023-07-22T12:36:39,226 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 9bcb0fd7964e4f0bb3b288296e47f9aa 2023-07-22T12:36:39,231 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6df1dc4958d04bf09fc29f60ce3010ba 2023-07-22T12:36:39,237 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38161/tmp/wal/data/hbase/meta/1588230740/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2023-07-22T12:36:39,238 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:36:39,238 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-07-22T12:36:39,238 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-07-22T12:36:39,238 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-07-22T12:36:39,238 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-07-22T12:36:39,312 INFO [RS:0;jenkins-hbase3:39575 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,39575,1690029321362; all regions closed. 2023-07-22T12:36:39,321 DEBUG [RS:0;jenkins-hbase3:39575 {}] wal.AbstractFSWAL(1062): Moved 1 WAL file(s) to /tmp/wal/oldWALs 2023-07-22T12:36:39,321 INFO [RS:0;jenkins-hbase3:39575 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C39575%2C1690029321362.meta:.meta(num 1690029323689) 2023-07-22T12:36:39,328 DEBUG [RS:0;jenkins-hbase3:39575 {}] wal.AbstractFSWAL(1062): Moved 1 WAL file(s) to /tmp/wal/oldWALs 2023-07-22T12:36:39,328 INFO [RS:0;jenkins-hbase3:39575 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C39575%2C1690029321362:(num 1690029323121) 2023-07-22T12:36:39,328 DEBUG [RS:0;jenkins-hbase3:39575 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:36:39,328 INFO [RS:0;jenkins-hbase3:39575 {}] regionserver.LeaseManager(133): Closed leases 2023-07-22T12:36:39,329 INFO [RS:0;jenkins-hbase3:39575 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2023-07-22T12:36:39,329 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-07-22T12:36:39,332 INFO [RS:0;jenkins-hbase3:39575 {}] ipc.NettyRpcServer(213): Stopping server on /172.31.12.81:39575 2023-07-22T12:36:39,339 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39575-0x100006222d70005, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/2/rs/jenkins-hbase3.apache.org,39575,1690029321362 2023-07-22T12:36:39,340 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/rs 2023-07-22T12:36:39,343 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,39575,1690029321362] 2023-07-22T12:36:39,343 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,39575,1690029321362; numProcessing=1 2023-07-22T12:36:39,344 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(224): Node /2/draining/jenkins-hbase3.apache.org,39575,1690029321362 already deleted, retry=false 2023-07-22T12:36:39,344 INFO [RegionServerTracker-0 {}] master.ServerManager(569): Cluster shutdown set; jenkins-hbase3.apache.org,39575,1690029321362 expired; onlineServers=0 2023-07-22T12:36:39,344 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2513): ***** STOPPING region server 'jenkins-hbase3.apache.org,46439,1690029321204' ***** 2023-07-22T12:36:39,344 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2527): STOPPED: Cluster shutdown set; onlineServer=0 2023-07-22T12:36:39,346 DEBUG [M:0;jenkins-hbase3:46439 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ba79313, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-07-22T12:36:39,346 INFO [M:0;jenkins-hbase3:46439 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-07-22T12:36:39,372 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/2/master 2023-07-22T12:36:39,387 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-07-22T12:36:39,417 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Set watcher on znode that does not yet exist, /2/master 2023-07-22T12:36:39,446 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39575-0x100006222d70005, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-07-22T12:36:39,446 INFO [RS:0;jenkins-hbase3:39575 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,39575,1690029321362; zookeeper connection closed. 2023-07-22T12:36:39,446 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39575-0x100006222d70005, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-07-22T12:36:39,448 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@28597c13 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@28597c13 2023-07-22T12:36:39,450 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-07-22T12:36:39,450 INFO [M:0;jenkins-hbase3:46439 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@42db99be{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-07-22T12:36:39,451 INFO [M:0;jenkins-hbase3:46439 {}] server.AbstractConnector(383): Stopped ServerConnector@7977b13c{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-07-22T12:36:39,451 INFO [M:0;jenkins-hbase3:46439 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-07-22T12:36:39,452 INFO [M:0;jenkins-hbase3:46439 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@9a76e17{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-07-22T12:36:39,453 INFO [M:0;jenkins-hbase3:46439 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@7668c2d0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/hadoop.log.dir/,STOPPED} 2023-07-22T12:36:39,454 INFO [M:0;jenkins-hbase3:46439 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,46439,1690029321204 2023-07-22T12:36:39,454 INFO [M:0;jenkins-hbase3:46439 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,46439,1690029321204; all regions closed. 2023-07-22T12:36:39,454 DEBUG [M:0;jenkins-hbase3:46439 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:36:39,454 INFO [M:0;jenkins-hbase3:46439 {}] master.HMaster(1664): Stopping master jetty server 2023-07-22T12:36:39,455 INFO [M:0;jenkins-hbase3:46439 {}] server.AbstractConnector(383): Stopped ServerConnector@10c22c07{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-07-22T12:36:39,455 DEBUG [M:0;jenkins-hbase3:46439 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-07-22T12:36:39,455 INFO [M:0;jenkins-hbase3:46439 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-07-22T12:36:39,455 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-07-22T12:36:39,456 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1690029322898 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1690029322898,5,FailOnTimeoutGroup] 2023-07-22T12:36:39,456 DEBUG [M:0;jenkins-hbase3:46439 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5f7bc8f4 to 127.0.0.1:54609 2023-07-22T12:36:39,456 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1690029322898 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1690029322898,5,FailOnTimeoutGroup] 2023-07-22T12:36:39,456 DEBUG [M:0;jenkins-hbase3:46439 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:36:39,456 DEBUG [M:0;jenkins-hbase3:46439 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-07-22T12:36:39,456 INFO [M:0;jenkins-hbase3:46439 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-07-22T12:36:39,456 DEBUG [M:0;jenkins-hbase3:46439 {}] master.HMaster(1687): Stopping service threads 2023-07-22T12:36:39,456 INFO [M:0;jenkins-hbase3:46439 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-07-22T12:36:39,456 ERROR [M:0;jenkins-hbase3:46439 {}] procedure2.ProcedureExecutor(655): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] 2023-07-22T12:36:39,457 INFO [M:0;jenkins-hbase3:46439 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-07-22T12:36:39,457 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-07-22T12:36:39,457 DEBUG [M:0;jenkins-hbase3:46439 {}] zookeeper.ZKUtil(347): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Unable to get data of znode /2/master because node does not exist (not an error) 2023-07-22T12:36:39,458 WARN [M:0;jenkins-hbase3:46439 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-07-22T12:36:39,458 INFO [M:0;jenkins-hbase3:46439 {}] assignment.AssignmentManager(382): Stopping assignment manager 2023-07-22T12:36:39,458 INFO [M:0;jenkins-hbase3:46439 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-07-22T12:36:39,459 DEBUG [M:0;jenkins-hbase3:46439 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-07-22T12:36:39,474 INFO [M:0;jenkins-hbase3:46439 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:36:39,474 DEBUG [M:0;jenkins-hbase3:46439 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:36:39,474 DEBUG [M:0;jenkins-hbase3:46439 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-07-22T12:36:39,474 DEBUG [M:0;jenkins-hbase3:46439 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:36:39,474 INFO [M:0;jenkins-hbase3:46439 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=25.76 KB heapSize=33.30 KB 2023-07-22T12:36:39,498 DEBUG [M:0;jenkins-hbase3:46439 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9a6a19f40cbd43258f057c301d698f8b is 88, key is hbase:meta,,1/info:sn/1690029323512/Put/seqid=0 2023-07-22T12:36:39,903 INFO [M:0;jenkins-hbase3:46439 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9a6a19f40cbd43258f057c301d698f8b 2023-07-22T12:36:39,912 DEBUG [M:0;jenkins-hbase3:46439 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 9a6a19f40cbd43258f057c301d698f8b 2023-07-22T12:36:39,932 DEBUG [M:0;jenkins-hbase3:46439 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c3c36e8ce52049038ae5e4d9ff47b088 is 241, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1690029323752/Put/seqid=0 2023-07-22T12:36:39,948 INFO [M:0;jenkins-hbase3:46439 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25.17 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c3c36e8ce52049038ae5e4d9ff47b088 2023-07-22T12:36:39,954 INFO [M:0;jenkins-hbase3:46439 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c3c36e8ce52049038ae5e4d9ff47b088 2023-07-22T12:36:39,954 DEBUG [M:0;jenkins-hbase3:46439 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: c3c36e8ce52049038ae5e4d9ff47b088 2023-07-22T12:36:39,967 DEBUG [M:0;jenkins-hbase3:46439 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a9b1d3cbd1a34773af0c53e2a8f9669f is 82, key is jenkins-hbase3.apache.org,39575,1690029321362/rs:state/1690029322942/Put/seqid=0 2023-07-22T12:36:39,972 INFO [M:0;jenkins-hbase3:46439 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a9b1d3cbd1a34773af0c53e2a8f9669f 2023-07-22T12:36:39,977 DEBUG [M:0;jenkins-hbase3:46439 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a9b1d3cbd1a34773af0c53e2a8f9669f 2023-07-22T12:36:39,978 DEBUG [M:0;jenkins-hbase3:46439 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9a6a19f40cbd43258f057c301d698f8b as hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9a6a19f40cbd43258f057c301d698f8b 2023-07-22T12:36:39,983 INFO [M:0;jenkins-hbase3:46439 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9a6a19f40cbd43258f057c301d698f8b, entries=8, sequenceid=76, filesize=5.5 K 2023-07-22T12:36:39,984 DEBUG [M:0;jenkins-hbase3:46439 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c3c36e8ce52049038ae5e4d9ff47b088 as hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c3c36e8ce52049038ae5e4d9ff47b088 2023-07-22T12:36:39,989 INFO [M:0;jenkins-hbase3:46439 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c3c36e8ce52049038ae5e4d9ff47b088 2023-07-22T12:36:39,989 INFO [M:0;jenkins-hbase3:46439 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c3c36e8ce52049038ae5e4d9ff47b088, entries=8, sequenceid=76, filesize=5.4 K 2023-07-22T12:36:39,990 DEBUG [M:0;jenkins-hbase3:46439 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a9b1d3cbd1a34773af0c53e2a8f9669f as hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a9b1d3cbd1a34773af0c53e2a8f9669f 2023-07-22T12:36:39,995 INFO [M:0;jenkins-hbase3:46439 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38161/user/jenkins/test-data/d93b6436-29f7-f96a-7c5f-206e0da9cb5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a9b1d3cbd1a34773af0c53e2a8f9669f, entries=1, sequenceid=76, filesize=5.1 K 2023-07-22T12:36:39,996 INFO [M:0;jenkins-hbase3:46439 {}] regionserver.HRegion(3022): Finished flush of dataSize ~25.76 KB/26381, heapSize ~33.01 KB/33800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 522ms, sequenceid=76, compaction requested=false 2023-07-22T12:36:39,999 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 9a6a19f40cbd43258f057c301d698f8b 2023-07-22T12:36:40,003 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-proc-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: c3c36e8ce52049038ae5e4d9ff47b088 2023-07-22T12:36:40,006 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-rs-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a9b1d3cbd1a34773af0c53e2a8f9669f 2023-07-22T12:36:40,007 INFO [M:0;jenkins-hbase3:46439 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:36:40,007 DEBUG [M:0;jenkins-hbase3:46439 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-07-22T12:36:40,011 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-07-22T12:36:40,011 INFO [M:0;jenkins-hbase3:46439 {}] flush.MasterFlushTableProcedureManager(81): stop: server shutting down. 2023-07-22T12:36:40,011 INFO [M:0;jenkins-hbase3:46439 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-07-22T12:36:40,012 INFO [M:0;jenkins-hbase3:46439 {}] ipc.NettyRpcServer(213): Stopping server on /172.31.12.81:46439 2023-07-22T12:36:40,014 DEBUG [M:0;jenkins-hbase3:46439 {}] zookeeper.RecoverableZooKeeper(224): Node /2/rs/jenkins-hbase3.apache.org,46439,1690029321204 already deleted, retry=false 2023-07-22T12:36:40,117 INFO [M:0;jenkins-hbase3:46439 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,46439,1690029321204; zookeeper connection closed. 2023-07-22T12:36:40,117 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-07-22T12:36:40,117 DEBUG [Listener at localhost/42321-EventThread {}] zookeeper.ZKWatcher(604): master:46439-0x100006222d70004, quorum=127.0.0.1:54609, baseZNode=/2 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-07-22T12:36:40,121 WARN [Listener at jenkins-hbase3.apache.org/37741 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-07-22T12:36:40,133 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-07-22T12:36:40,241 WARN [BP-1026815893-172.31.12.81-1690029319799 heartbeating to localhost/127.0.0.1:38161 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-07-22T12:36:40,241 WARN [BP-1026815893-172.31.12.81-1690029319799 heartbeating to localhost/127.0.0.1:38161 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-1026815893-172.31.12.81-1690029319799 (Datanode Uuid a99da7a0-b8a1-481d-bfd6-37121c4b89bf) service to localhost/127.0.0.1:38161 2023-07-22T12:36:40,243 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/cluster_3623d274-aa27-74cf-d526-f07dcf62976b/dfs/data/data5/current/BP-1026815893-172.31.12.81-1690029319799 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-07-22T12:36:40,244 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/cluster_3623d274-aa27-74cf-d526-f07dcf62976b/dfs/data/data6/current/BP-1026815893-172.31.12.81-1690029319799 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-07-22T12:36:40,249 WARN [Listener at jenkins-hbase3.apache.org/37741 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-07-22T12:36:40,258 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-07-22T12:36:40,369 WARN [BP-1026815893-172.31.12.81-1690029319799 heartbeating to localhost/127.0.0.1:38161 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-07-22T12:36:40,369 WARN [BP-1026815893-172.31.12.81-1690029319799 heartbeating to localhost/127.0.0.1:38161 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-1026815893-172.31.12.81-1690029319799 (Datanode Uuid 04e3e809-64d1-429d-8b2c-007ae22876f6) service to localhost/127.0.0.1:38161 2023-07-22T12:36:40,370 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/cluster_3623d274-aa27-74cf-d526-f07dcf62976b/dfs/data/data3/current/BP-1026815893-172.31.12.81-1690029319799 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-07-22T12:36:40,370 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/cluster_3623d274-aa27-74cf-d526-f07dcf62976b/dfs/data/data4/current/BP-1026815893-172.31.12.81-1690029319799 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-07-22T12:36:40,379 WARN [Listener at jenkins-hbase3.apache.org/37741 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-07-22T12:36:40,395 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-07-22T12:36:40,501 WARN [BP-1026815893-172.31.12.81-1690029319799 heartbeating to localhost/127.0.0.1:38161 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-07-22T12:36:40,501 WARN [BP-1026815893-172.31.12.81-1690029319799 heartbeating to localhost/127.0.0.1:38161 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-1026815893-172.31.12.81-1690029319799 (Datanode Uuid 6e1b725d-9d4d-4c66-9493-606eafe4ca96) service to localhost/127.0.0.1:38161 2023-07-22T12:36:40,502 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/cluster_3623d274-aa27-74cf-d526-f07dcf62976b/dfs/data/data1/current/BP-1026815893-172.31.12.81-1690029319799 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-07-22T12:36:40,502 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0840b4c-2bb2-b91a-8228-76ef8291fb5a/cluster_3623d274-aa27-74cf-d526-f07dcf62976b/dfs/data/data2/current/BP-1026815893-172.31.12.81-1690029319799 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-07-22T12:36:40,552 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-07-22T12:36:40,698 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-07-22T12:36:40,698 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-07-22T12:36:40,698 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] client.ConnectionImplementation(2095): Closing master protocol: MasterService 2023-07-22T12:36:40,698 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x443737b3 to 127.0.0.1:54609 2023-07-22T12:36:40,698 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:36:40,698 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-07-22T12:36:40,698 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] util.JVMClusterUtil(257): Found active master hash=995269432, stopped=false 2023-07-22T12:36:40,699 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] master.ServerManager(910): Cluster shutdown requested of master=jenkins-hbase3.apache.org,46137,1690029311532 2023-07-22T12:36:40,700 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2023-07-22T12:36:40,700 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2023-07-22T12:36:40,700 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:36:40,700 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] procedure2.ProcedureExecutor(631): Stopping 2023-07-22T12:36:40,700 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:36:40,701 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-07-22T12:36:40,701 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-07-22T12:36:40,701 DEBUG [Listener at jenkins-hbase3.apache.org/37741 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:36:40,702 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] regionserver.HRegionServer(2513): ***** STOPPING region server 'jenkins-hbase3.apache.org,38147,1690029314593' ***** 2023-07-22T12:36:40,702 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] regionserver.HRegionServer(2527): STOPPED: Shutdown requested 2023-07-22T12:36:40,702 INFO [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-07-22T12:36:40,707 INFO [RS:0;jenkins-hbase3:38147 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@771961df{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-07-22T12:36:40,707 INFO [RS:0;jenkins-hbase3:38147 {}] server.AbstractConnector(383): Stopped ServerConnector@1167861e{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-07-22T12:36:40,707 INFO [RS:0;jenkins-hbase3:38147 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-07-22T12:36:40,709 INFO [RS:0;jenkins-hbase3:38147 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@132f9d7f{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-07-22T12:36:40,711 INFO [RS:0;jenkins-hbase3:38147 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@11566dea{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/hadoop.log.dir/,STOPPED} 2023-07-22T12:36:40,712 INFO [RS:0;jenkins-hbase3:38147 {}] regionserver.HeapMemoryManager(220): Stopping 2023-07-22T12:36:40,712 INFO [RS:0;jenkins-hbase3:38147 {}] flush.RegionServerFlushTableProcedureManager(117): Stopping region server flush procedure manager gracefully. 2023-07-22T12:36:40,712 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-07-22T12:36:40,712 INFO [RS:0;jenkins-hbase3:38147 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-07-22T12:36:40,712 INFO [RS:0;jenkins-hbase3:38147 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-07-22T12:36:40,712 INFO [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(3524): Received CLOSE for 678d09e574bb66f63f5cd63b46064d40 2023-07-22T12:36:40,712 INFO [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(3524): Received CLOSE for de72a5b968421854d430063920aa35db 2023-07-22T12:36:40,712 INFO [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(3524): Received CLOSE for 30c77692ccc4e0df458e1f12cc34f3f8 2023-07-22T12:36:40,712 INFO [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(3524): Received CLOSE for e2f9453920ffaa2bfd87a3743468e74e 2023-07-22T12:36:40,713 INFO [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(3524): Received CLOSE for 8f674da8b67afb2bfe7cd3f65c30fe3f 2023-07-22T12:36:40,713 INFO [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(3524): Received CLOSE for 5c9301698cb41dfb7fd9be1c9c6a1f19 2023-07-22T12:36:40,713 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 678d09e574bb66f63f5cd63b46064d40, disabling compactions & flushes 2023-07-22T12:36:40,713 INFO [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(3524): Received CLOSE for 55c1ed70a7e1f74fa5c816e79d65220e 2023-07-22T12:36:40,713 INFO [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:36:40,713 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1690029339611.678d09e574bb66f63f5cd63b46064d40. 2023-07-22T12:36:40,714 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1690029339611.678d09e574bb66f63f5cd63b46064d40. 2023-07-22T12:36:40,714 INFO [RS:0;jenkins-hbase3:38147 {}] client.ConnectionImplementation(2095): Closing master protocol: MasterService 2023-07-22T12:36:40,714 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1690029339611.678d09e574bb66f63f5cd63b46064d40. after waiting 0 ms 2023-07-22T12:36:40,714 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1690029339611.678d09e574bb66f63f5cd63b46064d40. 2023-07-22T12:36:40,714 DEBUG [RS:0;jenkins-hbase3:38147 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:36:40,714 INFO [RS:0;jenkins-hbase3:38147 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-07-22T12:36:40,714 INFO [RS:0;jenkins-hbase3:38147 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-07-22T12:36:40,714 INFO [RS:0;jenkins-hbase3:38147 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-07-22T12:36:40,714 INFO [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(3524): Received CLOSE for 1588230740 2023-07-22T12:36:40,714 INFO [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(1579): Waiting on 8 regions to close 2023-07-22T12:36:40,714 DEBUG [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(1583): Online Regions={678d09e574bb66f63f5cd63b46064d40=backup:system_bulk,,1690029339611.678d09e574bb66f63f5cd63b46064d40., de72a5b968421854d430063920aa35db=hbase:namespace,,1690029318288.de72a5b968421854d430063920aa35db., 30c77692ccc4e0df458e1f12cc34f3f8=ns3:test-16900293305672,,1690029334007.30c77692ccc4e0df458e1f12cc34f3f8., e2f9453920ffaa2bfd87a3743468e74e=test-1690029330567,,1690029331075.e2f9453920ffaa2bfd87a3743468e74e., 8f674da8b67afb2bfe7cd3f65c30fe3f=ns2:test-16900293305671,,1690029333313.8f674da8b67afb2bfe7cd3f65c30fe3f., 1588230740=hbase:meta,,1.1588230740, 5c9301698cb41dfb7fd9be1c9c6a1f19=backup:system,,1690029338484.5c9301698cb41dfb7fd9be1c9c6a1f19., 55c1ed70a7e1f74fa5c816e79d65220e=ns4:test-16900293305673,,1690029336131.55c1ed70a7e1f74fa5c816e79d65220e.} 2023-07-22T12:36:40,716 DEBUG [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 30c77692ccc4e0df458e1f12cc34f3f8, 55c1ed70a7e1f74fa5c816e79d65220e, 5c9301698cb41dfb7fd9be1c9c6a1f19, 678d09e574bb66f63f5cd63b46064d40, 8f674da8b67afb2bfe7cd3f65c30fe3f, de72a5b968421854d430063920aa35db, e2f9453920ffaa2bfd87a3743468e74e 2023-07-22T12:36:40,716 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-07-22T12:36:40,716 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-07-22T12:36:40,716 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-07-22T12:36:40,716 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-07-22T12:36:40,716 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-07-22T12:36:40,716 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=13.51 KB heapSize=24 KB 2023-07-22T12:36:40,720 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system_bulk/678d09e574bb66f63f5cd63b46064d40/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-07-22T12:36:40,720 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:36:40,723 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system_bulk,,1690029339611.678d09e574bb66f63f5cd63b46064d40. 2023-07-22T12:36:40,723 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 678d09e574bb66f63f5cd63b46064d40: 2023-07-22T12:36:40,723 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1690029339611.678d09e574bb66f63f5cd63b46064d40. 2023-07-22T12:36:40,725 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing de72a5b968421854d430063920aa35db, disabling compactions & flushes 2023-07-22T12:36:40,725 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1690029318288.de72a5b968421854d430063920aa35db. 2023-07-22T12:36:40,725 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1690029318288.de72a5b968421854d430063920aa35db. 2023-07-22T12:36:40,725 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1690029318288.de72a5b968421854d430063920aa35db. after waiting 0 ms 2023-07-22T12:36:40,725 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1690029318288.de72a5b968421854d430063920aa35db. 2023-07-22T12:36:40,725 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing de72a5b968421854d430063920aa35db 1/1 column families, dataSize=249 B heapSize=1.02 KB 2023-07-22T12:36:40,736 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/hbase/meta/1588230740/.tmp/info/378c152ef2d8492984b58bc887fddad4 is 159, key is ns2:test-16900293305671,,1690029333313.8f674da8b67afb2bfe7cd3f65c30fe3f./info:regioninfo/1690029333685/Put/seqid=0 2023-07-22T12:36:40,743 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/hbase/namespace/de72a5b968421854d430063920aa35db/.tmp/info/8d8cae9831cf45e38cc002c01036271d is 45, key is default/info:d/1690029319663/Put/seqid=0 2023-07-22T12:36:40,744 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.12 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/hbase/meta/1588230740/.tmp/info/378c152ef2d8492984b58bc887fddad4 2023-07-22T12:36:40,751 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 378c152ef2d8492984b58bc887fddad4 2023-07-22T12:36:40,752 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/hbase/namespace/de72a5b968421854d430063920aa35db/.tmp/info/8d8cae9831cf45e38cc002c01036271d 2023-07-22T12:36:40,759 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8d8cae9831cf45e38cc002c01036271d 2023-07-22T12:36:40,760 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/hbase/namespace/de72a5b968421854d430063920aa35db/.tmp/info/8d8cae9831cf45e38cc002c01036271d as hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/hbase/namespace/de72a5b968421854d430063920aa35db/info/8d8cae9831cf45e38cc002c01036271d 2023-07-22T12:36:40,763 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-07-22T12:36:40,765 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/hbase/namespace/de72a5b968421854d430063920aa35db/info/8d8cae9831cf45e38cc002c01036271d, entries=7, sequenceid=11, filesize=5.1 K 2023-07-22T12:36:40,765 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~249 B/249, heapSize ~1.01 KB/1032, currentSize=0 B/0 for de72a5b968421854d430063920aa35db in 40ms, sequenceid=11, compaction requested=false 2023-07-22T12:36:40,768 DEBUG [StoreFileCloser-de72a5b968421854d430063920aa35db-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8d8cae9831cf45e38cc002c01036271d 2023-07-22T12:36:40,774 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/hbase/namespace/de72a5b968421854d430063920aa35db/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2023-07-22T12:36:40,774 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:36:40,776 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1690029318288.de72a5b968421854d430063920aa35db. 2023-07-22T12:36:40,776 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for de72a5b968421854d430063920aa35db: 2023-07-22T12:36:40,776 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1690029318288.de72a5b968421854d430063920aa35db. 2023-07-22T12:36:40,777 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 30c77692ccc4e0df458e1f12cc34f3f8, disabling compactions & flushes 2023-07-22T12:36:40,777 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns3:test-16900293305672,,1690029334007.30c77692ccc4e0df458e1f12cc34f3f8. 2023-07-22T12:36:40,777 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16900293305672,,1690029334007.30c77692ccc4e0df458e1f12cc34f3f8. 2023-07-22T12:36:40,777 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16900293305672,,1690029334007.30c77692ccc4e0df458e1f12cc34f3f8. after waiting 0 ms 2023-07-22T12:36:40,777 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16900293305672,,1690029334007.30c77692ccc4e0df458e1f12cc34f3f8. 2023-07-22T12:36:40,778 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/hbase/meta/1588230740/.tmp/rep_barrier/d612adf496c04b8693236f3f672d7b1d is 90, key is table1,,1690029353811.da5ae8ac17989cb406336333d6113a9e./rep_barrier:/1690029359145/DeleteFamily/seqid=0 2023-07-22T12:36:40,782 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/ns3/test-16900293305672/30c77692ccc4e0df458e1f12cc34f3f8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-07-22T12:36:40,783 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:36:40,784 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns3:test-16900293305672,,1690029334007.30c77692ccc4e0df458e1f12cc34f3f8. 2023-07-22T12:36:40,784 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 30c77692ccc4e0df458e1f12cc34f3f8: 2023-07-22T12:36:40,785 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-16900293305672,,1690029334007.30c77692ccc4e0df458e1f12cc34f3f8. 2023-07-22T12:36:40,785 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing e2f9453920ffaa2bfd87a3743468e74e, disabling compactions & flushes 2023-07-22T12:36:40,785 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region test-1690029330567,,1690029331075.e2f9453920ffaa2bfd87a3743468e74e. 2023-07-22T12:36:40,785 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1690029330567,,1690029331075.e2f9453920ffaa2bfd87a3743468e74e. 2023-07-22T12:36:40,785 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on test-1690029330567,,1690029331075.e2f9453920ffaa2bfd87a3743468e74e. after waiting 0 ms 2023-07-22T12:36:40,785 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region test-1690029330567,,1690029331075.e2f9453920ffaa2bfd87a3743468e74e. 2023-07-22T12:36:40,790 DEBUG [StoreFileCloser-e2f9453920ffaa2bfd87a3743468e74e-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a663b1eb7c49421a9a8af7e19a9900bf 2023-07-22T12:36:40,795 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/default/test-1690029330567/e2f9453920ffaa2bfd87a3743468e74e/recovered.edits/107.seqid, newMaxSeqId=107, maxSeqId=1 2023-07-22T12:36:40,795 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:36:40,797 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed test-1690029330567,,1690029331075.e2f9453920ffaa2bfd87a3743468e74e. 2023-07-22T12:36:40,797 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for e2f9453920ffaa2bfd87a3743468e74e: 2023-07-22T12:36:40,797 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1690029330567,,1690029331075.e2f9453920ffaa2bfd87a3743468e74e. 2023-07-22T12:36:40,798 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 8f674da8b67afb2bfe7cd3f65c30fe3f, disabling compactions & flushes 2023-07-22T12:36:40,798 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns2:test-16900293305671,,1690029333313.8f674da8b67afb2bfe7cd3f65c30fe3f. 2023-07-22T12:36:40,798 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16900293305671,,1690029333313.8f674da8b67afb2bfe7cd3f65c30fe3f. 2023-07-22T12:36:40,798 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16900293305671,,1690029333313.8f674da8b67afb2bfe7cd3f65c30fe3f. after waiting 0 ms 2023-07-22T12:36:40,798 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16900293305671,,1690029333313.8f674da8b67afb2bfe7cd3f65c30fe3f. 2023-07-22T12:36:40,798 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 8f674da8b67afb2bfe7cd3f65c30fe3f 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-07-22T12:36:40,814 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/ns2/test-16900293305671/8f674da8b67afb2bfe7cd3f65c30fe3f/.tmp/f/393bae7e112e45aaa321f4fbd8469856 is 37, key is row10/f:q1/1690029333938/Put/seqid=0 2023-07-22T12:36:40,916 DEBUG [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 55c1ed70a7e1f74fa5c816e79d65220e, 5c9301698cb41dfb7fd9be1c9c6a1f19, 8f674da8b67afb2bfe7cd3f65c30fe3f 2023-07-22T12:36:41,116 DEBUG [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 55c1ed70a7e1f74fa5c816e79d65220e, 5c9301698cb41dfb7fd9be1c9c6a1f19, 8f674da8b67afb2bfe7cd3f65c30fe3f 2023-07-22T12:36:41,186 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=172 B at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/hbase/meta/1588230740/.tmp/rep_barrier/d612adf496c04b8693236f3f672d7b1d 2023-07-22T12:36:41,193 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: d612adf496c04b8693236f3f672d7b1d 2023-07-22T12:36:41,214 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/hbase/meta/1588230740/.tmp/table/8b3f8dc58096417f84d2ceff5c09ba7b is 84, key is table1,,1690029353811.da5ae8ac17989cb406336333d6113a9e./table:/1690029359145/DeleteFamily/seqid=0 2023-07-22T12:36:41,225 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/ns2/test-16900293305671/8f674da8b67afb2bfe7cd3f65c30fe3f/.tmp/f/393bae7e112e45aaa321f4fbd8469856 2023-07-22T12:36:41,231 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 393bae7e112e45aaa321f4fbd8469856 2023-07-22T12:36:41,232 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/ns2/test-16900293305671/8f674da8b67afb2bfe7cd3f65c30fe3f/.tmp/f/393bae7e112e45aaa321f4fbd8469856 as hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/ns2/test-16900293305671/8f674da8b67afb2bfe7cd3f65c30fe3f/f/393bae7e112e45aaa321f4fbd8469856 2023-07-22T12:36:41,237 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/ns2/test-16900293305671/8f674da8b67afb2bfe7cd3f65c30fe3f/f/393bae7e112e45aaa321f4fbd8469856, entries=99, sequenceid=103, filesize=8.2 K 2023-07-22T12:36:41,239 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 8f674da8b67afb2bfe7cd3f65c30fe3f in 440ms, sequenceid=103, compaction requested=false 2023-07-22T12:36:41,244 DEBUG [StoreFileCloser-8f674da8b67afb2bfe7cd3f65c30fe3f-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 393bae7e112e45aaa321f4fbd8469856 2023-07-22T12:36:41,249 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/ns2/test-16900293305671/8f674da8b67afb2bfe7cd3f65c30fe3f/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-07-22T12:36:41,250 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:36:41,251 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns2:test-16900293305671,,1690029333313.8f674da8b67afb2bfe7cd3f65c30fe3f. 2023-07-22T12:36:41,251 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 8f674da8b67afb2bfe7cd3f65c30fe3f: 2023-07-22T12:36:41,251 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-16900293305671,,1690029333313.8f674da8b67afb2bfe7cd3f65c30fe3f. 2023-07-22T12:36:41,251 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 5c9301698cb41dfb7fd9be1c9c6a1f19, disabling compactions & flushes 2023-07-22T12:36:41,251 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system,,1690029338484.5c9301698cb41dfb7fd9be1c9c6a1f19. 2023-07-22T12:36:41,251 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1690029338484.5c9301698cb41dfb7fd9be1c9c6a1f19. 2023-07-22T12:36:41,251 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1690029338484.5c9301698cb41dfb7fd9be1c9c6a1f19. after waiting 0 ms 2023-07-22T12:36:41,251 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1690029338484.5c9301698cb41dfb7fd9be1c9c6a1f19. 2023-07-22T12:36:41,251 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 5c9301698cb41dfb7fd9be1c9c6a1f19 2/2 column families, dataSize=985 B heapSize=1.97 KB 2023-07-22T12:36:41,256 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/.tmp/meta/3ae8e97e657a4cceb29ad77d0191459e is 169, key is trslm:hdfs://localhost:38161/backupUT\x00test-1690029330567/meta:log-roll-map/1690029370401/Put/seqid=0 2023-07-22T12:36:41,317 DEBUG [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 55c1ed70a7e1f74fa5c816e79d65220e, 5c9301698cb41dfb7fd9be1c9c6a1f19 2023-07-22T12:36:41,517 DEBUG [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 55c1ed70a7e1f74fa5c816e79d65220e, 5c9301698cb41dfb7fd9be1c9c6a1f19 2023-07-22T12:36:41,583 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-07-22T12:36:41,583 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-07-22T12:36:41,629 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.22 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/hbase/meta/1588230740/.tmp/table/8b3f8dc58096417f84d2ceff5c09ba7b 2023-07-22T12:36:41,634 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8b3f8dc58096417f84d2ceff5c09ba7b 2023-07-22T12:36:41,635 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/hbase/meta/1588230740/.tmp/info/378c152ef2d8492984b58bc887fddad4 as hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/hbase/meta/1588230740/info/378c152ef2d8492984b58bc887fddad4 2023-07-22T12:36:41,640 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/hbase/meta/1588230740/info/378c152ef2d8492984b58bc887fddad4, entries=74, sequenceid=63, filesize=14.4 K 2023-07-22T12:36:41,641 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/hbase/meta/1588230740/.tmp/rep_barrier/d612adf496c04b8693236f3f672d7b1d as hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/hbase/meta/1588230740/rep_barrier/d612adf496c04b8693236f3f672d7b1d 2023-07-22T12:36:41,646 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/hbase/meta/1588230740/rep_barrier/d612adf496c04b8693236f3f672d7b1d, entries=2, sequenceid=63, filesize=5.4 K 2023-07-22T12:36:41,647 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/hbase/meta/1588230740/.tmp/table/8b3f8dc58096417f84d2ceff5c09ba7b as hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/hbase/meta/1588230740/table/8b3f8dc58096417f84d2ceff5c09ba7b 2023-07-22T12:36:41,652 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/hbase/meta/1588230740/table/8b3f8dc58096417f84d2ceff5c09ba7b, entries=17, sequenceid=63, filesize=6.0 K 2023-07-22T12:36:41,653 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~13.51 KB/13834, heapSize ~23.95 KB/24528, currentSize=0 B/0 for 1588230740 in 937ms, sequenceid=63, compaction requested=false 2023-07-22T12:36:41,653 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-07-22T12:36:41,656 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 378c152ef2d8492984b58bc887fddad4 2023-07-22T12:36:41,661 DEBUG [StoreFileCloser-1588230740-rep_barrier-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: d612adf496c04b8693236f3f672d7b1d 2023-07-22T12:36:41,666 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8b3f8dc58096417f84d2ceff5c09ba7b 2023-07-22T12:36:41,669 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=27 (bloomFilter=true), to=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/.tmp/meta/3ae8e97e657a4cceb29ad77d0191459e 2023-07-22T12:36:41,672 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/hbase/meta/1588230740/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=1 2023-07-22T12:36:41,673 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:36:41,673 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-07-22T12:36:41,675 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-07-22T12:36:41,675 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-07-22T12:36:41,675 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-07-22T12:36:41,676 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3ae8e97e657a4cceb29ad77d0191459e 2023-07-22T12:36:41,679 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/.tmp/session/3f7d2f41c2c64a519dfb600e0d4b190b is 310, key is session:backup_1690029359273/session:context/1690029370824/Put/seqid=0 2023-07-22T12:36:41,691 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-07-22T12:36:41,692 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-07-22T12:36:41,692 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-07-22T12:36:41,693 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16900293305672 2023-07-22T12:36:41,693 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-07-22T12:36:41,694 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16900293305671 2023-07-22T12:36:41,694 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1690029330567 2023-07-22T12:36:41,695 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2023-07-22T12:36:41,718 INFO [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(1579): Waiting on 2 regions to close 2023-07-22T12:36:41,718 DEBUG [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(1583): Online Regions={5c9301698cb41dfb7fd9be1c9c6a1f19=backup:system,,1690029338484.5c9301698cb41dfb7fd9be1c9c6a1f19., 55c1ed70a7e1f74fa5c816e79d65220e=ns4:test-16900293305673,,1690029336131.55c1ed70a7e1f74fa5c816e79d65220e.} 2023-07-22T12:36:41,718 DEBUG [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(1609): Waiting on 55c1ed70a7e1f74fa5c816e79d65220e, 5c9301698cb41dfb7fd9be1c9c6a1f19 2023-07-22T12:36:41,918 DEBUG [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(1609): Waiting on 55c1ed70a7e1f74fa5c816e79d65220e, 5c9301698cb41dfb7fd9be1c9c6a1f19 2023-07-22T12:36:42,084 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=534 B at sequenceid=27 (bloomFilter=true), to=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/.tmp/session/3f7d2f41c2c64a519dfb600e0d4b190b 2023-07-22T12:36:42,095 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3f7d2f41c2c64a519dfb600e0d4b190b 2023-07-22T12:36:42,096 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/.tmp/meta/3ae8e97e657a4cceb29ad77d0191459e as hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/meta/3ae8e97e657a4cceb29ad77d0191459e 2023-07-22T12:36:42,101 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/meta/3ae8e97e657a4cceb29ad77d0191459e, entries=4, sequenceid=27, filesize=5.5 K 2023-07-22T12:36:42,102 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/.tmp/session/3f7d2f41c2c64a519dfb600e0d4b190b as hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/session/3f7d2f41c2c64a519dfb600e0d4b190b 2023-07-22T12:36:42,107 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/session/3f7d2f41c2c64a519dfb600e0d4b190b, entries=2, sequenceid=27, filesize=5.3 K 2023-07-22T12:36:42,107 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~985 B/985, heapSize ~1.94 KB/1984, currentSize=0 B/0 for 5c9301698cb41dfb7fd9be1c9c6a1f19 in 856ms, sequenceid=27, compaction requested=true 2023-07-22T12:36:42,110 DEBUG [StoreFileCloser-5c9301698cb41dfb7fd9be1c9c6a1f19-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: e91b686b0ef14d2ca9fa2273c2792e2a 2023-07-22T12:36:42,113 DEBUG [StoreFileCloser-5c9301698cb41dfb7fd9be1c9c6a1f19-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3ae8e97e657a4cceb29ad77d0191459e 2023-07-22T12:36:42,116 DEBUG [StoreFileCloser-5c9301698cb41dfb7fd9be1c9c6a1f19-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: d3f88af37ddf42418c7fa151a0dd871a 2023-07-22T12:36:42,119 DEBUG [StoreFileCloser-5c9301698cb41dfb7fd9be1c9c6a1f19-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: eb02733a331149f6a13d862744b427bc 2023-07-22T12:36:42,122 DEBUG [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(1609): Waiting on 55c1ed70a7e1f74fa5c816e79d65220e, 5c9301698cb41dfb7fd9be1c9c6a1f19 2023-07-22T12:36:42,122 DEBUG [StoreFileCloser-5c9301698cb41dfb7fd9be1c9c6a1f19-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3f7d2f41c2c64a519dfb600e0d4b190b 2023-07-22T12:36:42,134 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/backup/system/5c9301698cb41dfb7fd9be1c9c6a1f19/recovered.edits/30.seqid, newMaxSeqId=30, maxSeqId=1 2023-07-22T12:36:42,136 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:36:42,137 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system,,1690029338484.5c9301698cb41dfb7fd9be1c9c6a1f19. 2023-07-22T12:36:42,137 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 5c9301698cb41dfb7fd9be1c9c6a1f19: 2023-07-22T12:36:42,137 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1690029338484.5c9301698cb41dfb7fd9be1c9c6a1f19. 2023-07-22T12:36:42,138 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 55c1ed70a7e1f74fa5c816e79d65220e, disabling compactions & flushes 2023-07-22T12:36:42,138 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns4:test-16900293305673,,1690029336131.55c1ed70a7e1f74fa5c816e79d65220e. 2023-07-22T12:36:42,138 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16900293305673,,1690029336131.55c1ed70a7e1f74fa5c816e79d65220e. 2023-07-22T12:36:42,139 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16900293305673,,1690029336131.55c1ed70a7e1f74fa5c816e79d65220e. after waiting 0 ms 2023-07-22T12:36:42,139 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16900293305673,,1690029336131.55c1ed70a7e1f74fa5c816e79d65220e. 2023-07-22T12:36:42,142 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/data/ns4/test-16900293305673/55c1ed70a7e1f74fa5c816e79d65220e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-07-22T12:36:42,142 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:36:42,144 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns4:test-16900293305673,,1690029336131.55c1ed70a7e1f74fa5c816e79d65220e. 2023-07-22T12:36:42,144 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 55c1ed70a7e1f74fa5c816e79d65220e: 2023-07-22T12:36:42,144 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-16900293305673,,1690029336131.55c1ed70a7e1f74fa5c816e79d65220e. 2023-07-22T12:36:42,225 INFO [Socket Reader #1 for port 0 {}] ipc.Server$Connection(2001): Auth successful for appattempt_1690029373888_0001_000001 (auth:SIMPLE) 2023-07-22T12:36:42,322 INFO [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,38147,1690029314593; all regions closed. 2023-07-22T12:36:42,331 DEBUG [RS:0;jenkins-hbase3:38147 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/oldWALs 2023-07-22T12:36:42,331 INFO [RS:0;jenkins-hbase3:38147 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C38147%2C1690029314593.meta:.meta(num 1690029363442) 2023-07-22T12:36:42,338 DEBUG [RS:0;jenkins-hbase3:38147 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/oldWALs 2023-07-22T12:36:42,338 INFO [RS:0;jenkins-hbase3:38147 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C38147%2C1690029314593:(num 1690029363460) 2023-07-22T12:36:42,338 DEBUG [RS:0;jenkins-hbase3:38147 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:36:42,338 INFO [RS:0;jenkins-hbase3:38147 {}] regionserver.LeaseManager(133): Closed leases 2023-07-22T12:36:42,338 INFO [RS:0;jenkins-hbase3:38147 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2023-07-22T12:36:42,339 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-07-22T12:36:42,341 INFO [RS:0;jenkins-hbase3:38147 {}] ipc.NettyRpcServer(213): Stopping server on /172.31.12.81:38147 2023-07-22T12:36:42,347 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rs/jenkins-hbase3.apache.org,38147,1690029314593 2023-07-22T12:36:42,347 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-07-22T12:36:42,357 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,38147,1690029314593] 2023-07-22T12:36:42,357 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,38147,1690029314593; numProcessing=1 2023-07-22T12:36:42,358 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(224): Node /1/draining/jenkins-hbase3.apache.org,38147,1690029314593 already deleted, retry=false 2023-07-22T12:36:42,358 INFO [RegionServerTracker-0 {}] master.ServerManager(569): Cluster shutdown set; jenkins-hbase3.apache.org,38147,1690029314593 expired; onlineServers=0 2023-07-22T12:36:42,358 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2513): ***** STOPPING region server 'jenkins-hbase3.apache.org,46137,1690029311532' ***** 2023-07-22T12:36:42,358 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2527): STOPPED: Cluster shutdown set; onlineServer=0 2023-07-22T12:36:42,360 DEBUG [M:0;jenkins-hbase3:46137 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a904658, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-07-22T12:36:42,360 INFO [M:0;jenkins-hbase3:46137 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-07-22T12:36:42,377 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/master 2023-07-22T12:36:42,377 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:36:42,388 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-07-22T12:36:42,397 INFO [M:0;jenkins-hbase3:46137 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@42323d37{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-07-22T12:36:42,398 INFO [M:0;jenkins-hbase3:46137 {}] server.AbstractConnector(383): Stopped ServerConnector@33d7be8d{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-07-22T12:36:42,399 INFO [M:0;jenkins-hbase3:46137 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-07-22T12:36:42,400 INFO [M:0;jenkins-hbase3:46137 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@1997ec4d{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-07-22T12:36:42,401 INFO [M:0;jenkins-hbase3:46137 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@65657a93{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/hadoop.log.dir/,STOPPED} 2023-07-22T12:36:42,402 INFO [M:0;jenkins-hbase3:46137 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,46137,1690029311532 2023-07-22T12:36:42,402 INFO [M:0;jenkins-hbase3:46137 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,46137,1690029311532; all regions closed. 2023-07-22T12:36:42,402 DEBUG [M:0;jenkins-hbase3:46137 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:36:42,402 INFO [M:0;jenkins-hbase3:46137 {}] master.HMaster(1664): Stopping master jetty server 2023-07-22T12:36:42,403 INFO [M:0;jenkins-hbase3:46137 {}] server.AbstractConnector(383): Stopped ServerConnector@427cb4e2{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-07-22T12:36:42,403 DEBUG [M:0;jenkins-hbase3:46137 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-07-22T12:36:42,403 INFO [M:0;jenkins-hbase3:46137 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-07-22T12:36:42,403 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-07-22T12:36:42,403 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1690029316449 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1690029316449,5,FailOnTimeoutGroup] 2023-07-22T12:36:42,403 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1690029316449 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1690029316449,5,FailOnTimeoutGroup] 2023-07-22T12:36:42,403 DEBUG [M:0;jenkins-hbase3:46137 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x12c3adfa to 127.0.0.1:54609 2023-07-22T12:36:42,403 DEBUG [M:0;jenkins-hbase3:46137 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:36:42,404 DEBUG [M:0;jenkins-hbase3:46137 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-07-22T12:36:42,404 INFO [M:0;jenkins-hbase3:46137 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-07-22T12:36:42,404 DEBUG [M:0;jenkins-hbase3:46137 {}] master.HMaster(1687): Stopping service threads 2023-07-22T12:36:42,404 INFO [M:0;jenkins-hbase3:46137 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-07-22T12:36:42,404 ERROR [M:0;jenkins-hbase3:46137 {}] procedure2.ProcedureExecutor(655): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-2,5,PEWorkerGroup] 2023-07-22T12:36:42,404 INFO [M:0;jenkins-hbase3:46137 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-07-22T12:36:42,404 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-07-22T12:36:42,405 DEBUG [M:0;jenkins-hbase3:46137 {}] zookeeper.ZKUtil(347): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Unable to get data of znode /1/master because node does not exist (not an error) 2023-07-22T12:36:42,405 WARN [M:0;jenkins-hbase3:46137 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-07-22T12:36:42,405 INFO [M:0;jenkins-hbase3:46137 {}] assignment.AssignmentManager(382): Stopping assignment manager 2023-07-22T12:36:42,405 INFO [M:0;jenkins-hbase3:46137 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-07-22T12:36:42,407 DEBUG [M:0;jenkins-hbase3:46137 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-07-22T12:36:42,418 INFO [M:0;jenkins-hbase3:46137 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:36:42,418 DEBUG [M:0;jenkins-hbase3:46137 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:36:42,418 DEBUG [M:0;jenkins-hbase3:46137 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-07-22T12:36:42,419 DEBUG [M:0;jenkins-hbase3:46137 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:36:42,419 INFO [M:0;jenkins-hbase3:46137 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=203.20 KB heapSize=244.09 KB 2023-07-22T12:36:42,432 DEBUG [M:0;jenkins-hbase3:46137 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9e5dc9e95af144b1b75275b6ec7be196 is 88, key is hbase:meta,,1/info:sn/1690029317485/Put/seqid=0 2023-07-22T12:36:42,458 INFO [RS:0;jenkins-hbase3:38147 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,38147,1690029314593; zookeeper connection closed. 2023-07-22T12:36:42,458 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-07-22T12:36:42,458 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38147-0x100006222d70001, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-07-22T12:36:42,458 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@461a3058 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@461a3058 2023-07-22T12:36:42,458 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-07-22T12:36:42,837 INFO [M:0;jenkins-hbase3:46137 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9e5dc9e95af144b1b75275b6ec7be196 2023-07-22T12:36:42,843 DEBUG [M:0;jenkins-hbase3:46137 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 9e5dc9e95af144b1b75275b6ec7be196 2023-07-22T12:36:42,857 DEBUG [M:0;jenkins-hbase3:46137 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cffd6e25ebfb4e5bbf806fe20d140a44 is 988, key is \x00\x00\x00\x00\x00\x00\x00\x1D/proc:d/1690029340796/Put/seqid=0 2023-07-22T12:36:43,264 INFO [M:0;jenkins-hbase3:46137 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=202.60 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cffd6e25ebfb4e5bbf806fe20d140a44 2023-07-22T12:36:43,269 INFO [M:0;jenkins-hbase3:46137 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cffd6e25ebfb4e5bbf806fe20d140a44 2023-07-22T12:36:43,269 DEBUG [M:0;jenkins-hbase3:46137 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: cffd6e25ebfb4e5bbf806fe20d140a44 2023-07-22T12:36:43,282 DEBUG [M:0;jenkins-hbase3:46137 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cad8ccbf732946809d2ec91a6688cf28 is 82, key is jenkins-hbase3.apache.org,38147,1690029314593/rs:state/1690029316485/Put/seqid=0 2023-07-22T12:36:43,686 INFO [M:0;jenkins-hbase3:46137 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cad8ccbf732946809d2ec91a6688cf28 2023-07-22T12:36:43,691 DEBUG [M:0;jenkins-hbase3:46137 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: cad8ccbf732946809d2ec91a6688cf28 2023-07-22T12:36:43,692 DEBUG [M:0;jenkins-hbase3:46137 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9e5dc9e95af144b1b75275b6ec7be196 as hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9e5dc9e95af144b1b75275b6ec7be196 2023-07-22T12:36:43,697 INFO [M:0;jenkins-hbase3:46137 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9e5dc9e95af144b1b75275b6ec7be196, entries=8, sequenceid=480, filesize=5.5 K 2023-07-22T12:36:43,698 DEBUG [M:0;jenkins-hbase3:46137 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cffd6e25ebfb4e5bbf806fe20d140a44 as hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cffd6e25ebfb4e5bbf806fe20d140a44 2023-07-22T12:36:43,703 INFO [M:0;jenkins-hbase3:46137 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cffd6e25ebfb4e5bbf806fe20d140a44 2023-07-22T12:36:43,703 INFO [M:0;jenkins-hbase3:46137 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cffd6e25ebfb4e5bbf806fe20d140a44, entries=53, sequenceid=480, filesize=16.7 K 2023-07-22T12:36:43,704 DEBUG [M:0;jenkins-hbase3:46137 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cad8ccbf732946809d2ec91a6688cf28 as hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cad8ccbf732946809d2ec91a6688cf28 2023-07-22T12:36:43,708 INFO [M:0;jenkins-hbase3:46137 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38685/user/jenkins/test-data/6a95d461-5863-3d77-d2b3-af21a6d4bb5f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cad8ccbf732946809d2ec91a6688cf28, entries=1, sequenceid=480, filesize=5.1 K 2023-07-22T12:36:43,709 INFO [M:0;jenkins-hbase3:46137 {}] regionserver.HRegion(3022): Finished flush of dataSize ~203.20 KB/208075, heapSize ~243.79 KB/249640, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1290ms, sequenceid=480, compaction requested=false 2023-07-22T12:36:43,715 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 9e5dc9e95af144b1b75275b6ec7be196 2023-07-22T12:36:43,716 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-proc-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: cffd6e25ebfb4e5bbf806fe20d140a44 2023-07-22T12:36:43,718 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-rs-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: cad8ccbf732946809d2ec91a6688cf28 2023-07-22T12:36:43,719 INFO [M:0;jenkins-hbase3:46137 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:36:43,719 DEBUG [M:0;jenkins-hbase3:46137 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-07-22T12:36:43,724 INFO [M:0;jenkins-hbase3:46137 {}] flush.MasterFlushTableProcedureManager(81): stop: server shutting down. 2023-07-22T12:36:43,724 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-07-22T12:36:43,724 INFO [M:0;jenkins-hbase3:46137 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-07-22T12:36:43,725 INFO [M:0;jenkins-hbase3:46137 {}] ipc.NettyRpcServer(213): Stopping server on /172.31.12.81:46137 2023-07-22T12:36:43,727 DEBUG [M:0;jenkins-hbase3:46137 {}] zookeeper.RecoverableZooKeeper(224): Node /1/rs/jenkins-hbase3.apache.org,46137,1690029311532 already deleted, retry=false 2023-07-22T12:36:43,829 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-07-22T12:36:43,829 INFO [M:0;jenkins-hbase3:46137 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,46137,1690029311532; zookeeper connection closed. 2023-07-22T12:36:43,829 DEBUG [Listener at localhost/39493-EventThread {}] zookeeper.ZKWatcher(604): master:46137-0x100006222d70000, quorum=127.0.0.1:54609, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-07-22T12:36:43,833 WARN [Listener at jenkins-hbase3.apache.org/37741 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-07-22T12:36:43,836 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-07-22T12:36:43,944 WARN [BP-1697032271-172.31.12.81-1690029308490 heartbeating to localhost/127.0.0.1:38685 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-07-22T12:36:43,944 WARN [BP-1697032271-172.31.12.81-1690029308490 heartbeating to localhost/127.0.0.1:38685 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-1697032271-172.31.12.81-1690029308490 (Datanode Uuid ebfe7662-6a3f-4444-8812-84efc237aad3) service to localhost/127.0.0.1:38685 2023-07-22T12:36:43,945 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/cluster_81664056-bd3e-8e52-f121-db92c3ef923e/dfs/data/data1/current/BP-1697032271-172.31.12.81-1690029308490 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-07-22T12:36:43,945 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/41c69e48-15e3-dab0-c854-81ccc3827e93/cluster_81664056-bd3e-8e52-f121-db92c3ef923e/dfs/data/data2/current/BP-1697032271-172.31.12.81-1690029308490 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-07-22T12:36:43,994 INFO [Finalizer {}] client.ConnectionImplementation(2095): Closing master protocol: MasterService 2023-07-22T12:36:43,994 DEBUG [Finalizer {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1320e970 to 127.0.0.1:54609 2023-07-22T12:36:43,994 DEBUG [Finalizer {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:36:43,999 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-07-22T12:36:44,065 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:36:44,286 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2023-07-22T12:36:44,314 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-07-22T12:36:44,314 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] hbase.HBaseTestingUtility(2857): Stopping mini mapreduce cluster... 2023-07-22T12:36:44,328 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-07-22T12:36:44,449 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-07-22T12:36:44,563 ERROR [Thread[Thread-493,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-07-22T12:36:44,564 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-07-22T12:36:44,670 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2023-07-22T12:36:44,677 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(61): Returning, interrupted : java.lang.InterruptedException 2023-07-22T12:36:44,679 ERROR [Thread[Thread-504,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-07-22T12:36:44,689 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-07-22T12:36:44,793 ERROR [Thread[Thread-468,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-07-22T12:36:44,794 INFO [Listener at jenkins-hbase3.apache.org/37741 {}] hbase.HBaseTestingUtility(2860): Mini mapreduce cluster stopped