2023-11-12 09:29:52,826 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@229f66ed 2023-11-12 09:29:52,841 main DEBUG Took 0.011965 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-11-12 09:29:52,841 main DEBUG PluginManager 'Core' found 129 plugins 2023-11-12 09:29:52,841 main DEBUG PluginManager 'Level' found 0 plugins 2023-11-12 09:29:52,842 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-11-12 09:29:52,843 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:29:52,852 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-11-12 09:29:52,864 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:29:52,866 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:29:52,867 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:29:52,867 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:29:52,867 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:29:52,867 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:29:52,869 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:29:52,869 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:29:52,869 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:29:52,870 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:29:52,871 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:29:52,871 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:29:52,871 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:29:52,872 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:29:52,872 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:29:52,873 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:29:52,873 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:29:52,873 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:29:52,874 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:29:52,874 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:29:52,874 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:29:52,875 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:29:52,875 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:29:52,876 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:29:52,876 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:29:52,877 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-11-12 09:29:52,879 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:29:52,881 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-11-12 09:29:52,882 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-11-12 09:29:52,883 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-11-12 09:29:52,884 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-11-12 09:29:52,885 main DEBUG PluginManager 'Converter' found 47 plugins 2023-11-12 09:29:52,900 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-11-12 09:29:52,903 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-11-12 09:29:52,908 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-11-12 09:29:52,908 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-11-12 09:29:52,909 main DEBUG createAppenders(={Console}) 2023-11-12 09:29:52,910 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@229f66ed initialized 2023-11-12 09:29:52,910 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@229f66ed 2023-11-12 09:29:52,910 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@229f66ed OK. 2023-11-12 09:29:52,911 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-11-12 09:29:52,911 main DEBUG OutputStream closed 2023-11-12 09:29:52,911 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-11-12 09:29:52,911 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-11-12 09:29:52,911 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@498d318c OK 2023-11-12 09:29:52,954 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-11-12 09:29:52,956 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-11-12 09:29:52,957 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-11-12 09:29:52,959 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-11-12 09:29:52,959 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-11-12 09:29:52,960 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-11-12 09:29:52,960 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-11-12 09:29:52,960 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-11-12 09:29:52,960 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-11-12 09:29:52,961 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-11-12 09:29:52,961 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-11-12 09:29:52,961 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-11-12 09:29:52,962 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-11-12 09:29:52,962 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-11-12 09:29:52,962 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-11-12 09:29:52,963 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-11-12 09:29:52,963 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-11-12 09:29:52,964 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-11-12 09:29:52,967 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-11-12 09:29:52,967 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4d5b6aac) with optional ClassLoader: null 2023-11-12 09:29:52,967 main DEBUG Shutdown hook enabled. Registering a new one. 2023-11-12 09:29:52,968 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@4d5b6aac] started OK. 2023-11-12T09:29:53,002 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestRemoteRestore timeout: 13 mins 2023-11-12 09:29:53,006 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-11-12 09:29:53,006 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-11-12T09:29:53,325 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27 2023-11-12T09:29:53,330 DEBUG [Time-limited test {}] impl.BackupManager(127): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-11-12T09:29:53,332 DEBUG [Time-limited test {}] impl.BackupManager(157): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:29:53,334 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-11-12T09:29:53,337 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/cluster_1b67588a-7841-a8ad-b22d-cb19219656d1, deleteOnExit=true 2023-11-12T09:29:53,337 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-11-12T09:29:53,338 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/test.cache.data in system properties and HBase conf 2023-11-12T09:29:53,338 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/hadoop.tmp.dir in system properties and HBase conf 2023-11-12T09:29:53,339 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/hadoop.log.dir in system properties and HBase conf 2023-11-12T09:29:53,339 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/mapreduce.cluster.local.dir in system properties and HBase conf 2023-11-12T09:29:53,339 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-11-12T09:29:53,339 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-11-12T09:29:53,451 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-11-12T09:29:53,979 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-11-12T09:29:53,984 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-11-12T09:29:53,984 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-11-12T09:29:53,984 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-11-12T09:29:53,985 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-12T09:29:53,985 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-11-12T09:29:53,985 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-11-12T09:29:53,986 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-12T09:29:53,986 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-12T09:29:53,986 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-11-12T09:29:53,987 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/nfs.dump.dir in system properties and HBase conf 2023-11-12T09:29:53,987 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/java.io.tmpdir in system properties and HBase conf 2023-11-12T09:29:53,987 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-12T09:29:53,987 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-11-12T09:29:53,988 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-11-12T09:29:54,501 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-11-12T09:29:54,505 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-11-12T09:29:54,809 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-11-12T09:29:54,959 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@75d6ed9b via org.mortbay.log.Slf4jLog 2023-11-12T09:29:54,974 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:29:55,017 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:29:55,054 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/java.io.tmpdir/Jetty_localhost_44099_hdfs____.w3aiyu/webapp 2023-11-12T09:29:55,193 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:44099 2023-11-12T09:29:55,215 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-11-12T09:29:55,215 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-11-12T09:29:55,783 WARN [Listener at localhost/38625 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-11-12T09:29:55,846 WARN [Listener at localhost/38625 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-11-12T09:29:55,865 WARN [Listener at localhost/38625 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:29:55,872 INFO [Listener at localhost/38625 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:29:55,878 INFO [Listener at localhost/38625 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/java.io.tmpdir/Jetty_localhost_37921_datanode____.7lapc5/webapp 2023-11-12T09:29:55,975 INFO [Listener at localhost/38625 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:37921 2023-11-12T09:29:56,254 WARN [Listener at localhost/33247 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-11-12T09:29:56,531 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-11-12T09:29:56,720 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xb6e3f9ec2b0ec410: Processing first storage report for DS-4bdce35f-16ea-409e-9d44-e8db6ed957eb from datanode 188c7465-4427-4b07-a6c4-b29194f7812c 2023-11-12T09:29:56,721 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xb6e3f9ec2b0ec410: from storage DS-4bdce35f-16ea-409e-9d44-e8db6ed957eb node DatanodeRegistration(127.0.0.1:44263, datanodeUuid=188c7465-4427-4b07-a6c4-b29194f7812c, infoPort=38139, infoSecurePort=0, ipcPort=33247, storageInfo=lv=-57;cid=testClusterID;nsid=1968955691;c=1699781394581), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2023-11-12T09:29:56,722 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xb6e3f9ec2b0ec410: Processing first storage report for DS-51f0036a-238e-4427-ab27-27293008abe2 from datanode 188c7465-4427-4b07-a6c4-b29194f7812c 2023-11-12T09:29:56,722 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xb6e3f9ec2b0ec410: from storage DS-51f0036a-238e-4427-ab27-27293008abe2 node DatanodeRegistration(127.0.0.1:44263, datanodeUuid=188c7465-4427-4b07-a6c4-b29194f7812c, infoPort=38139, infoSecurePort=0, ipcPort=33247, storageInfo=lv=-57;cid=testClusterID;nsid=1968955691;c=1699781394581), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-11-12T09:29:56,761 DEBUG [Listener at localhost/33247 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27 2023-11-12T09:29:56,901 INFO [Listener at localhost/33247 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/cluster_1b67588a-7841-a8ad-b22d-cb19219656d1/zookeeper_0, clientPort=57951, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/cluster_1b67588a-7841-a8ad-b22d-cb19219656d1/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/cluster_1b67588a-7841-a8ad-b22d-cb19219656d1/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2023-11-12T09:29:56,920 INFO [Listener at localhost/33247 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=57951 2023-11-12T09:29:56,928 INFO [Listener at localhost/33247 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:29:56,931 INFO [Listener at localhost/33247 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:29:57,600 INFO [Listener at localhost/33247 {}] util.FSUtils(462): Created version file at hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a with version=8 2023-11-12T09:29:57,600 INFO [Listener at localhost/33247 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/hbase-staging 2023-11-12T09:29:57,617 DEBUG [Listener at localhost/33247 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-11-12T09:29:57,617 DEBUG [Listener at localhost/33247 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-11-12T09:29:57,617 DEBUG [Listener at localhost/33247 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-11-12T09:29:57,617 DEBUG [Listener at localhost/33247 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-11-12T09:29:57,827 DEBUG [Listener at localhost/33247 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-11-12T09:29:58,591 INFO [Listener at localhost/33247 {}] client.ConnectionUtils(128): master/jenkins-hbase3:0 server-side Connection retries=45 2023-11-12T09:29:58,637 INFO [Listener at localhost/33247 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-12T09:29:58,638 INFO [Listener at localhost/33247 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-11-12T09:29:58,638 INFO [Listener at localhost/33247 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-11-12T09:29:58,638 INFO [Listener at localhost/33247 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-12T09:29:58,638 INFO [Listener at localhost/33247 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-11-12T09:29:58,790 INFO [Listener at localhost/33247 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-11-12T09:29:58,920 INFO [Listener at localhost/33247 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-11-12T09:29:58,939 DEBUG [Listener at localhost/33247 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-11-12T09:29:58,945 INFO [Listener at localhost/33247 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-11-12T09:29:58,994 DEBUG [Listener at localhost/33247 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 24557 (auto-detected) 2023-11-12T09:29:58,995 DEBUG [Listener at localhost/33247 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:0f:ff:fe:94:aa:bf (auto-detected) 2023-11-12T09:29:59,038 INFO [Listener at localhost/33247 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:44289 2023-11-12T09:29:59,053 INFO [Listener at localhost/33247 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:29:59,055 INFO [Listener at localhost/33247 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:29:59,076 INFO [Listener at localhost/33247 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:44289 connecting to ZooKeeper ensemble=127.0.0.1:57951 2023-11-12T09:29:59,131 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:442890x0, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-11-12T09:29:59,135 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:44289-0x1004cf7b7840000 connected 2023-11-12T09:29:59,168 DEBUG [Listener at localhost/33247 {}] zookeeper.ZKUtil(113): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-11-12T09:29:59,171 DEBUG [Listener at localhost/33247 {}] zookeeper.ZKUtil(113): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-11-12T09:29:59,175 DEBUG [Listener at localhost/33247 {}] zookeeper.ZKUtil(113): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-11-12T09:29:59,186 DEBUG [Listener at localhost/33247 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44289 2023-11-12T09:29:59,187 DEBUG [Listener at localhost/33247 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44289 2023-11-12T09:29:59,187 DEBUG [Listener at localhost/33247 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44289 2023-11-12T09:29:59,188 DEBUG [Listener at localhost/33247 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44289 2023-11-12T09:29:59,188 DEBUG [Listener at localhost/33247 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44289 2023-11-12T09:29:59,244 INFO [Listener at localhost/33247 {}] log.Log(170): Logging initialized @7154ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2023-11-12T09:29:59,366 INFO [Listener at localhost/33247 {}] http.HttpServer(981): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-11-12T09:29:59,367 INFO [Listener at localhost/33247 {}] http.HttpServer(981): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-11-12T09:29:59,368 INFO [Listener at localhost/33247 {}] http.HttpServer(981): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-11-12T09:29:59,370 INFO [Listener at localhost/33247 {}] http.HttpServer(960): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-11-12T09:29:59,370 INFO [Listener at localhost/33247 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-11-12T09:29:59,370 INFO [Listener at localhost/33247 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-11-12T09:29:59,373 INFO [Listener at localhost/33247 {}] http.HttpServer(805): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-11-12T09:29:59,529 INFO [Listener at localhost/33247 {}] http.HttpServer(1227): Jetty bound to port 34023 2023-11-12T09:29:59,532 INFO [Listener at localhost/33247 {}] server.Server(375): jetty-9.4.52.v20230823; built: 2023-08-23T19:29:37.669Z; git: abdcda73818a1a2c705da276edb0bf6581e7997e; jvm 1.8.0_362-b09 2023-11-12T09:29:59,574 INFO [Listener at localhost/33247 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:29:59,577 INFO [Listener at localhost/33247 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@2505f075{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/hadoop.log.dir/,AVAILABLE} 2023-11-12T09:29:59,578 INFO [Listener at localhost/33247 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:29:59,578 INFO [Listener at localhost/33247 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@8701aa9{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-11-12T09:29:59,643 INFO [Listener at localhost/33247 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-11-12T09:29:59,656 INFO [Listener at localhost/33247 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-11-12T09:29:59,656 INFO [Listener at localhost/33247 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-11-12T09:29:59,658 INFO [Listener at localhost/33247 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-11-12T09:29:59,665 INFO [Listener at localhost/33247 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:30:00,865 INFO [Listener at localhost/33247 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@613d33f5{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-11-12T09:30:00,876 INFO [Listener at localhost/33247 {}] server.AbstractConnector(333): Started ServerConnector@17469d4d{HTTP/1.1, (http/1.1)}{0.0.0.0:34023} 2023-11-12T09:30:00,877 INFO [Listener at localhost/33247 {}] server.Server(415): Started @8786ms 2023-11-12T09:30:00,880 INFO [Listener at localhost/33247 {}] master.HMaster(486): hbase.rootdir=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a, hbase.cluster.distributed=false 2023-11-12T09:30:00,944 INFO [Listener at localhost/33247 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase3:0 server-side Connection retries=45 2023-11-12T09:30:00,944 INFO [Listener at localhost/33247 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-12T09:30:00,944 INFO [Listener at localhost/33247 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-11-12T09:30:00,944 INFO [Listener at localhost/33247 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-11-12T09:30:00,944 INFO [Listener at localhost/33247 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-12T09:30:00,944 INFO [Listener at localhost/33247 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-11-12T09:30:00,953 INFO [Listener at localhost/33247 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-11-12T09:30:00,955 INFO [Listener at localhost/33247 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-11-12T09:30:00,956 INFO [Listener at localhost/33247 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:37573 2023-11-12T09:30:00,959 INFO [Listener at localhost/33247 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-11-12T09:30:00,966 DEBUG [Listener at localhost/33247 {}] mob.MobFileCache(121): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-11-12T09:30:00,967 INFO [Listener at localhost/33247 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:30:00,971 INFO [Listener at localhost/33247 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:30:00,973 INFO [Listener at localhost/33247 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:37573 connecting to ZooKeeper ensemble=127.0.0.1:57951 2023-11-12T09:30:00,977 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): regionserver:375730x0, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-11-12T09:30:00,978 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:37573-0x1004cf7b7840001 connected 2023-11-12T09:30:00,978 DEBUG [Listener at localhost/33247 {}] zookeeper.ZKUtil(113): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-11-12T09:30:00,979 DEBUG [Listener at localhost/33247 {}] zookeeper.ZKUtil(113): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-11-12T09:30:00,980 DEBUG [Listener at localhost/33247 {}] zookeeper.ZKUtil(113): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-11-12T09:30:00,981 DEBUG [Listener at localhost/33247 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37573 2023-11-12T09:30:00,981 DEBUG [Listener at localhost/33247 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37573 2023-11-12T09:30:00,982 DEBUG [Listener at localhost/33247 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37573 2023-11-12T09:30:00,983 DEBUG [Listener at localhost/33247 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37573 2023-11-12T09:30:00,984 DEBUG [Listener at localhost/33247 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37573 2023-11-12T09:30:00,988 INFO [Listener at localhost/33247 {}] http.HttpServer(981): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-11-12T09:30:00,988 INFO [Listener at localhost/33247 {}] http.HttpServer(981): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-11-12T09:30:00,989 INFO [Listener at localhost/33247 {}] http.HttpServer(981): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-11-12T09:30:00,990 INFO [Listener at localhost/33247 {}] http.HttpServer(960): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-11-12T09:30:00,990 INFO [Listener at localhost/33247 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-11-12T09:30:00,990 INFO [Listener at localhost/33247 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-11-12T09:30:00,990 INFO [Listener at localhost/33247 {}] http.HttpServer(805): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-11-12T09:30:00,993 INFO [Listener at localhost/33247 {}] http.HttpServer(1227): Jetty bound to port 42251 2023-11-12T09:30:00,993 INFO [Listener at localhost/33247 {}] server.Server(375): jetty-9.4.52.v20230823; built: 2023-08-23T19:29:37.669Z; git: abdcda73818a1a2c705da276edb0bf6581e7997e; jvm 1.8.0_362-b09 2023-11-12T09:30:00,995 INFO [Listener at localhost/33247 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:30:00,995 INFO [Listener at localhost/33247 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@be88332{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/hadoop.log.dir/,AVAILABLE} 2023-11-12T09:30:00,996 INFO [Listener at localhost/33247 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:30:00,996 INFO [Listener at localhost/33247 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@5892ebca{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-11-12T09:30:01,009 INFO [Listener at localhost/33247 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-11-12T09:30:01,010 INFO [Listener at localhost/33247 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-11-12T09:30:01,010 INFO [Listener at localhost/33247 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-11-12T09:30:01,011 INFO [Listener at localhost/33247 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2023-11-12T09:30:01,012 INFO [Listener at localhost/33247 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:30:01,017 INFO [Listener at localhost/33247 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@559a367f{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-11-12T09:30:01,018 INFO [Listener at localhost/33247 {}] server.AbstractConnector(333): Started ServerConnector@220910c3{HTTP/1.1, (http/1.1)}{0.0.0.0:42251} 2023-11-12T09:30:01,018 INFO [Listener at localhost/33247 {}] server.Server(415): Started @8928ms 2023-11-12T09:30:01,023 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.52.v20230823; built: 2023-08-23T19:29:37.669Z; git: abdcda73818a1a2c705da276edb0bf6581e7997e; jvm 1.8.0_362-b09 2023-11-12T09:30:01,050 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@31bf9364{HTTP/1.1, (http/1.1)}{0.0.0.0:41907} 2023-11-12T09:30:01,050 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(415): Started @8959ms 2023-11-12T09:30:01,050 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2389): Adding backup master ZNode /1/backup-masters/jenkins-hbase3.apache.org,44289,1699781397811 2023-11-12T09:30:01,069 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-12T09:30:01,069 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-12T09:30:01,072 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/jenkins-hbase3.apache.org,44289,1699781397811 2023-11-12T09:30:01,072 DEBUG [M:0;jenkins-hbase3:44289 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:44289 2023-11-12T09:30:01,091 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-11-12T09:30:01,091 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-11-12T09:30:01,091 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:30:01,091 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:30:01,092 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Set watcher on existing znode=/1/master 2023-11-12T09:30:01,094 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/jenkins-hbase3.apache.org,44289,1699781397811 from backup master directory 2023-11-12T09:30:01,094 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Set watcher on existing znode=/1/master 2023-11-12T09:30:01,098 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/jenkins-hbase3.apache.org,44289,1699781397811 2023-11-12T09:30:01,098 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-12T09:30:01,099 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-12T09:30:01,099 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-11-12T09:30:01,099 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,44289,1699781397811 2023-11-12T09:30:01,103 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-11-12T09:30:01,104 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-11-12T09:30:01,190 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(611): Created cluster ID file at hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/hbase.id with ID: 893bb15f-6e5d-452a-b5b8-5bb0a2a3bdbe 2023-11-12T09:30:01,229 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:30:01,245 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:30:01,245 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:30:01,287 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:30:01,289 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-11-12T09:30:01,308 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(271): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-11-12T09:30:01,308 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(277): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-11-12T09:30:01,309 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(286): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:284) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:311) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:952) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2414) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:575) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:572) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-12T09:30:01,319 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:952) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2414) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:575) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:572) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-12T09:30:01,321 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-12T09:30:01,391 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/MasterData/data/master/store 2023-11-12T09:30:01,420 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-11-12T09:30:01,420 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:30:01,421 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-11-12T09:30:01,421 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:30:01,421 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:30:01,421 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-11-12T09:30:01,422 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:30:01,422 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:30:01,422 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-11-12T09:30:01,424 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/MasterData/data/master/store/.initializing 2023-11-12T09:30:01,424 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/MasterData/WALs/jenkins-hbase3.apache.org,44289,1699781397811 2023-11-12T09:30:01,433 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-11-12T09:30:01,447 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C44289%2C1699781397811, suffix=, logDir=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/MasterData/WALs/jenkins-hbase3.apache.org,44289,1699781397811, archiveDir=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/MasterData/oldWALs, maxLogs=10 2023-11-12T09:30:01,475 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/MasterData/WALs/jenkins-hbase3.apache.org,44289,1699781397811/jenkins-hbase3.apache.org%2C44289%2C1699781397811.1699781401453, exclude list is [], retry=0 2023-11-12T09:30:01,504 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44263,DS-4bdce35f-16ea-409e-9d44-e8db6ed957eb,DISK] 2023-11-12T09:30:01,511 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:347) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:431) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:425) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:483) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:478) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:499) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:407) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.5.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-12T09:30:01,593 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/MasterData/WALs/jenkins-hbase3.apache.org,44289,1699781397811/jenkins-hbase3.apache.org%2C44289%2C1699781397811.1699781401453 2023-11-12T09:30:01,594 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:44263,DS-4bdce35f-16ea-409e-9d44-e8db6ed957eb,DISK]] 2023-11-12T09:30:01,594 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:30:01,595 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:30:01,599 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:30:01,601 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:30:01,668 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:30:01,708 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-11-12T09:30:01,714 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:01,718 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:30:01,718 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:30:01,721 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-11-12T09:30:01,721 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:01,722 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:30:01,722 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:30:01,725 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-11-12T09:30:01,725 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:01,726 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:30:01,726 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:30:01,728 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-11-12T09:30:01,728 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:01,729 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:30:01,734 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:30:01,735 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:30:01,750 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-11-12T09:30:01,753 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:30:01,759 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:30:01,760 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11377892800, jitterRate=0.05964884161949158}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-11-12T09:30:01,766 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-11-12T09:30:01,769 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-11-12T09:30:01,806 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ace4440, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:30:01,863 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(861): No meta location available on zookeeper, skip migrating... 2023-11-12T09:30:01,887 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-11-12T09:30:01,887 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(563): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-11-12T09:30:01,892 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-11-12T09:30:01,893 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(583): Recovered RegionProcedureStore lease in 1 msec 2023-11-12T09:30:01,902 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(597): Loaded RegionProcedureStore in 8 msec 2023-11-12T09:30:01,903 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-11-12T09:30:01,942 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-11-12T09:30:01,966 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2023-11-12T09:30:01,969 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/balancer already deleted, retry=false 2023-11-12T09:30:01,974 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-11-12T09:30:01,976 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2023-11-12T09:30:01,977 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/normalizer already deleted, retry=false 2023-11-12T09:30:01,981 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-11-12T09:30:01,993 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2023-11-12T09:30:01,996 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/split already deleted, retry=false 2023-11-12T09:30:01,997 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2023-11-12T09:30:01,999 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/merge already deleted, retry=false 2023-11-12T09:30:02,017 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2023-11-12T09:30:02,019 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/snapshot-cleanup already deleted, retry=false 2023-11-12T09:30:02,025 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-11-12T09:30:02,025 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-11-12T09:30:02,025 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:30:02,025 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:30:02,026 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(805): Active/primary master=jenkins-hbase3.apache.org,44289,1699781397811, sessionid=0x1004cf7b7840000, setting cluster-up flag (Was=false) 2023-11-12T09:30:02,036 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-11-12T09:30:02,053 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:30:02,053 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:30:02,060 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2023-11-12T09:30:02,062 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,44289,1699781397811 2023-11-12T09:30:02,066 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:30:02,066 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:30:02,073 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2023-11-12T09:30:02,074 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,44289,1699781397811 2023-11-12T09:30:02,079 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:30:02,079 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:30:02,085 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2023-11-12T09:30:02,087 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,44289,1699781397811 2023-11-12T09:30:02,132 DEBUG [RS:0;jenkins-hbase3:37573 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:37573 2023-11-12T09:30:02,133 INFO [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(997): ClusterId : 893bb15f-6e5d-452a-b5b8-5bb0a2a3bdbe 2023-11-12T09:30:02,135 INFO [RS:0;jenkins-hbase3:37573 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-11-12T09:30:02,138 DEBUG [RS:0;jenkins-hbase3:37573 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-11-12T09:30:02,144 DEBUG [RS:0;jenkins-hbase3:37573 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-11-12T09:30:02,144 DEBUG [RS:0;jenkins-hbase3:37573 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-11-12T09:30:02,148 DEBUG [RS:0;jenkins-hbase3:37573 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-11-12T09:30:02,148 DEBUG [RS:0;jenkins-hbase3:37573 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-11-12T09:30:02,151 DEBUG [RS:0;jenkins-hbase3:37573 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-11-12T09:30:02,152 DEBUG [RS:0;jenkins-hbase3:37573 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60ec1e70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:30:02,154 DEBUG [RS:0;jenkins-hbase3:37573 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2802f103, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-11-12T09:30:02,158 INFO [RS:0;jenkins-hbase3:37573 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-11-12T09:30:02,159 INFO [RS:0;jenkins-hbase3:37573 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-11-12T09:30:02,159 DEBUG [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-11-12T09:30:02,162 INFO [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,44289,1699781397811 with isa=jenkins-hbase3.apache.org/172.31.12.81:37573, startcode=1699781400943 2023-11-12T09:30:02,180 DEBUG [RS:0;jenkins-hbase3:37573 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-11-12T09:30:02,232 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-11-12T09:30:02,245 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-11-12T09:30:02,253 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-11-12T09:30:02,257 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:38649, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-11-12T09:30:02,265 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,44289,1699781397811 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-11-12T09:30:02,270 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44289 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3212) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:592) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17393) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-11-12T09:30:02,271 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-11-12T09:30:02,272 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-11-12T09:30:02,272 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-11-12T09:30:02,272 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-11-12T09:30:02,272 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-11-12T09:30:02,272 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:30:02,272 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-11-12T09:30:02,273 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:30:02,280 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1699781432280 2023-11-12T09:30:02,284 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-11-12T09:30:02,295 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-11-12T09:30:02,299 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-11-12T09:30:02,300 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-11-12T09:30:02,306 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:02,307 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-11-12T09:30:02,308 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-11-12T09:30:02,309 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-11-12T09:30:02,309 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-11-12T09:30:02,310 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-11-12T09:30:02,310 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-11-12T09:30:02,314 DEBUG [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(3046): Master is not running yet 2023-11-12T09:30:02,314 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-11-12T09:30:02,314 WARN [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(1088): reportForDuty failed; sleeping 100 ms and then retrying. 2023-11-12T09:30:02,316 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-11-12T09:30:02,319 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-11-12T09:30:02,319 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-11-12T09:30:02,333 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x50f72ab6 to 127.0.0.1:57951 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:30:02,335 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-11-12T09:30:02,335 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a 2023-11-12T09:30:02,345 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@711a6593, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:30:02,345 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-11-12T09:30:02,353 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-11-12T09:30:02,353 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:30:02,355 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-11-12T09:30:02,356 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-11-12T09:30:02,358 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1699781402357,5,FailOnTimeoutGroup] 2023-11-12T09:30:02,358 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1699781402358,5,FailOnTimeoutGroup] 2023-11-12T09:30:02,358 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-11-12T09:30:02,359 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1636): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-11-12T09:30:02,359 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-11-12T09:30:02,359 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:02,360 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:30:02,360 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-11-12T09:30:02,361 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-11-12T09:30:02,361 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-11-12T09:30:02,363 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-11-12T09:30:02,363 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:02,364 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:30:02,364 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-11-12T09:30:02,366 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-11-12T09:30:02,366 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:02,367 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:30:02,369 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/hbase/meta/1588230740 2023-11-12T09:30:02,369 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/hbase/meta/1588230740 2023-11-12T09:30:02,372 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-11-12T09:30:02,374 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-11-12T09:30:02,378 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:30:02,379 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10198965280, jitterRate=-0.05014733970165253}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-11-12T09:30:02,382 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-11-12T09:30:02,382 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-11-12T09:30:02,383 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-11-12T09:30:02,383 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-11-12T09:30:02,383 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-11-12T09:30:02,383 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-11-12T09:30:02,385 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-11-12T09:30:02,385 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-11-12T09:30:02,389 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-11-12T09:30:02,389 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-11-12T09:30:02,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-11-12T09:30:02,416 INFO [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,44289,1699781397811 with isa=jenkins-hbase3.apache.org/172.31.12.81:37573, startcode=1699781400943 2023-11-12T09:30:02,422 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-11-12T09:30:02,426 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44289 {}] master.ServerManager(396): Registering regionserver=jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:02,428 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-11-12T09:30:02,442 DEBUG [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a 2023-11-12T09:30:02,442 DEBUG [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:38625 2023-11-12T09:30:02,442 DEBUG [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=34023 2023-11-12T09:30:02,454 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-11-12T09:30:02,455 DEBUG [RS:0;jenkins-hbase3:37573 {}] zookeeper.ZKUtil(111): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Set watcher on existing znode=/1/rs/jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:02,455 WARN [RS:0;jenkins-hbase3:37573 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-11-12T09:30:02,456 INFO [RS:0;jenkins-hbase3:37573 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-12T09:30:02,456 DEBUG [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/WALs/jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:02,460 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,37573,1699781400943] 2023-11-12T09:30:02,480 DEBUG [RS:0;jenkins-hbase3:37573 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-11-12T09:30:02,495 INFO [RS:0;jenkins-hbase3:37573 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-11-12T09:30:02,510 INFO [RS:0;jenkins-hbase3:37573 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-11-12T09:30:02,514 INFO [RS:0;jenkins-hbase3:37573 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-11-12T09:30:02,514 INFO [RS:0;jenkins-hbase3:37573 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-11-12T09:30:02,515 INFO [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-11-12T09:30:02,525 INFO [RS:0;jenkins-hbase3:37573 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-11-12T09:30:02,525 DEBUG [RS:0;jenkins-hbase3:37573 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:30:02,525 DEBUG [RS:0;jenkins-hbase3:37573 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:30:02,525 DEBUG [RS:0;jenkins-hbase3:37573 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:30:02,525 DEBUG [RS:0;jenkins-hbase3:37573 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:30:02,525 DEBUG [RS:0;jenkins-hbase3:37573 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:30:02,526 DEBUG [RS:0;jenkins-hbase3:37573 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-11-12T09:30:02,526 DEBUG [RS:0;jenkins-hbase3:37573 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:30:02,526 DEBUG [RS:0;jenkins-hbase3:37573 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:30:02,526 DEBUG [RS:0;jenkins-hbase3:37573 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:30:02,526 DEBUG [RS:0;jenkins-hbase3:37573 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:30:02,526 DEBUG [RS:0;jenkins-hbase3:37573 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:30:02,526 DEBUG [RS:0;jenkins-hbase3:37573 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-11-12T09:30:02,526 DEBUG [RS:0;jenkins-hbase3:37573 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-11-12T09:30:02,527 INFO [RS:0;jenkins-hbase3:37573 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-11-12T09:30:02,528 INFO [RS:0;jenkins-hbase3:37573 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-11-12T09:30:02,528 INFO [RS:0;jenkins-hbase3:37573 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-11-12T09:30:02,528 INFO [RS:0;jenkins-hbase3:37573 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-11-12T09:30:02,528 INFO [RS:0;jenkins-hbase3:37573 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37573,1699781400943-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-11-12T09:30:02,543 INFO [RS:0;jenkins-hbase3:37573 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-11-12T09:30:02,547 INFO [RS:0;jenkins-hbase3:37573 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37573,1699781400943-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-11-12T09:30:02,564 INFO [RS:0;jenkins-hbase3:37573 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,37573,1699781400943 started 2023-11-12T09:30:02,564 INFO [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,37573,1699781400943, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:37573, sessionid=0x1004cf7b7840001 2023-11-12T09:30:02,565 DEBUG [RS:0;jenkins-hbase3:37573 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-11-12T09:30:02,565 DEBUG [RS:0;jenkins-hbase3:37573 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:02,565 DEBUG [RS:0;jenkins-hbase3:37573 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,37573,1699781400943' 2023-11-12T09:30:02,565 DEBUG [RS:0;jenkins-hbase3:37573 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2023-11-12T09:30:02,566 DEBUG [RS:0;jenkins-hbase3:37573 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2023-11-12T09:30:02,567 DEBUG [RS:0;jenkins-hbase3:37573 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-11-12T09:30:02,567 DEBUG [RS:0;jenkins-hbase3:37573 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-11-12T09:30:02,567 DEBUG [RS:0;jenkins-hbase3:37573 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,37573,1699781400943' 2023-11-12T09:30:02,567 DEBUG [RS:0;jenkins-hbase3:37573 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:30:02,568 DEBUG [RS:0;jenkins-hbase3:37573 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:30:02,569 INFO [RS:0;jenkins-hbase3:37573 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-11-12T09:30:02,569 DEBUG [RS:0;jenkins-hbase3:37573 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-11-12T09:30:02,569 DEBUG [RS:0;jenkins-hbase3:37573 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-11-12T09:30:02,569 DEBUG [RS:0;jenkins-hbase3:37573 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:02,569 DEBUG [RS:0;jenkins-hbase3:37573 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,37573,1699781400943' 2023-11-12T09:30:02,569 DEBUG [RS:0;jenkins-hbase3:37573 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2023-11-12T09:30:02,570 DEBUG [RS:0;jenkins-hbase3:37573 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2023-11-12T09:30:02,570 DEBUG [RS:0;jenkins-hbase3:37573 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-11-12T09:30:02,571 INFO [RS:0;jenkins-hbase3:37573 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-11-12T09:30:02,571 INFO [RS:0;jenkins-hbase3:37573 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-11-12T09:30:02,579 WARN [jenkins-hbase3:44289 {}] assignment.AssignmentManager(2298): No servers available; cannot place 1 unassigned regions. 2023-11-12T09:30:02,681 INFO [RS:0;jenkins-hbase3:37573 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-11-12T09:30:02,684 INFO [RS:0;jenkins-hbase3:37573 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C37573%2C1699781400943, suffix=, logDir=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/WALs/jenkins-hbase3.apache.org,37573,1699781400943, archiveDir=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/oldWALs, maxLogs=32 2023-11-12T09:30:02,696 DEBUG [RS:0;jenkins-hbase3:37573 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/WALs/jenkins-hbase3.apache.org,37573,1699781400943/jenkins-hbase3.apache.org%2C37573%2C1699781400943.1699781402686, exclude list is [], retry=0 2023-11-12T09:30:02,702 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44263,DS-4bdce35f-16ea-409e-9d44-e8db6ed957eb,DISK] 2023-11-12T09:30:02,706 INFO [RS:0;jenkins-hbase3:37573 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/WALs/jenkins-hbase3.apache.org,37573,1699781400943/jenkins-hbase3.apache.org%2C37573%2C1699781400943.1699781402686 2023-11-12T09:30:02,707 DEBUG [RS:0;jenkins-hbase3:37573 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:44263,DS-4bdce35f-16ea-409e-9d44-e8db6ed957eb,DISK]] 2023-11-12T09:30:02,830 DEBUG [jenkins-hbase3:44289 {}] assignment.AssignmentManager(2319): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-11-12T09:30:02,841 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:02,847 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,37573,1699781400943, state=OPENING 2023-11-12T09:30:02,856 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-11-12T09:30:02,857 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:30:02,857 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:30:02,858 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-12T09:30:02,858 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-12T09:30:02,862 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,37573,1699781400943}] 2023-11-12T09:30:03,011 INFO [AsyncFSWAL-0-hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/MasterData {}] wal.AbstractFSWAL(1174): Slow sync cost: 138 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44263,DS-4bdce35f-16ea-409e-9d44-e8db6ed957eb,DISK]] 2023-11-12T09:30:03,194 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:03,197 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2023-11-12T09:30:03,201 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:54256, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-11-12T09:30:03,213 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-11-12T09:30:03,214 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-12T09:30:03,215 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-11-12T09:30:03,218 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C37573%2C1699781400943.meta, suffix=.meta, logDir=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/WALs/jenkins-hbase3.apache.org,37573,1699781400943, archiveDir=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/oldWALs, maxLogs=32 2023-11-12T09:30:03,237 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/WALs/jenkins-hbase3.apache.org,37573,1699781400943/jenkins-hbase3.apache.org%2C37573%2C1699781400943.meta.1699781403220.meta, exclude list is [], retry=0 2023-11-12T09:30:03,241 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44263,DS-4bdce35f-16ea-409e-9d44-e8db6ed957eb,DISK] 2023-11-12T09:30:03,247 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/WALs/jenkins-hbase3.apache.org,37573,1699781400943/jenkins-hbase3.apache.org%2C37573%2C1699781400943.meta.1699781403220.meta 2023-11-12T09:30:03,247 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:44263,DS-4bdce35f-16ea-409e-9d44-e8db6ed957eb,DISK]] 2023-11-12T09:30:03,247 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:30:03,254 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:30:03,255 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-11-12T09:30:03,280 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-11-12T09:30:03,280 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-11-12T09:30:03,287 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-11-12T09:30:03,288 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:30:03,288 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-11-12T09:30:03,288 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-11-12T09:30:03,296 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-11-12T09:30:03,297 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-11-12T09:30:03,297 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:03,298 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:30:03,299 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-11-12T09:30:03,300 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-11-12T09:30:03,300 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:03,301 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:30:03,301 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-11-12T09:30:03,303 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-11-12T09:30:03,303 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:03,304 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:30:03,306 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/hbase/meta/1588230740 2023-11-12T09:30:03,308 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/hbase/meta/1588230740 2023-11-12T09:30:03,311 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-11-12T09:30:03,314 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-11-12T09:30:03,316 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11913673120, jitterRate=0.10954727232456207}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-11-12T09:30:03,317 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-11-12T09:30:03,335 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1699781403186 2023-11-12T09:30:03,355 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:meta,,1.1588230740 2023-11-12T09:30:03,356 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-11-12T09:30:03,357 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:03,360 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,37573,1699781400943, state=OPEN 2023-11-12T09:30:03,363 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-11-12T09:30:03,363 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-11-12T09:30:03,363 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-12T09:30:03,363 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-12T09:30:03,367 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=3, resume processing ppid=2 2023-11-12T09:30:03,368 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,37573,1699781400943 in 501 msec 2023-11-12T09:30:03,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=2, resume processing ppid=1 2023-11-12T09:30:03,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 967 msec 2023-11-12T09:30:03,384 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.2290 sec 2023-11-12T09:30:03,384 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1064): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1699781403384, completionTime=-1 2023-11-12T09:30:03,384 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(816): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-11-12T09:30:03,384 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1645): Joining cluster... 2023-11-12T09:30:03,460 DEBUG [hconnection-0x2c8ba79b-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:30:03,464 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:54260, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:30:03,481 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1657): Number of RegionServers=1 2023-11-12T09:30:03,481 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1699781463481 2023-11-12T09:30:03,481 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1699781523481 2023-11-12T09:30:03,482 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1664): Joined the cluster in 97 msec 2023-11-12T09:30:03,508 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,44289,1699781397811-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-11-12T09:30:03,508 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,44289,1699781397811-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-11-12T09:30:03,508 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,44289,1699781397811-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-11-12T09:30:03,510 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:44289, period=300000, unit=MILLISECONDS is enabled. 2023-11-12T09:30:03,511 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-11-12T09:30:03,517 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-11-12T09:30:03,531 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-11-12T09:30:03,532 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2369): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-11-12T09:30:03,542 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-11-12T09:30:03,546 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:30:03,548 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:03,550 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:30:03,968 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => daff3e7fcafd33ff24024100cbc1e9c9, NAME => 'hbase:namespace,,1699781403532.daff3e7fcafd33ff24024100cbc1e9c9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a 2023-11-12T09:30:03,979 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1699781403532.daff3e7fcafd33ff24024100cbc1e9c9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:30:03,979 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing daff3e7fcafd33ff24024100cbc1e9c9, disabling compactions & flushes 2023-11-12T09:30:03,979 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1699781403532.daff3e7fcafd33ff24024100cbc1e9c9. 2023-11-12T09:30:03,979 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1699781403532.daff3e7fcafd33ff24024100cbc1e9c9. 2023-11-12T09:30:03,979 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1699781403532.daff3e7fcafd33ff24024100cbc1e9c9. after waiting 0 ms 2023-11-12T09:30:03,979 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1699781403532.daff3e7fcafd33ff24024100cbc1e9c9. 2023-11-12T09:30:03,980 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1699781403532.daff3e7fcafd33ff24024100cbc1e9c9. 2023-11-12T09:30:03,980 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for daff3e7fcafd33ff24024100cbc1e9c9: 2023-11-12T09:30:03,982 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:30:03,997 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1699781403532.daff3e7fcafd33ff24024100cbc1e9c9.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1699781403984"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781403984"}]},"ts":"1699781403984"} 2023-11-12T09:30:04,032 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:30:04,034 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:30:04,040 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781404034"}]},"ts":"1699781404034"} 2023-11-12T09:30:04,044 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-11-12T09:30:04,049 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=daff3e7fcafd33ff24024100cbc1e9c9, ASSIGN}] 2023-11-12T09:30:04,051 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=daff3e7fcafd33ff24024100cbc1e9c9, ASSIGN 2023-11-12T09:30:04,053 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=daff3e7fcafd33ff24024100cbc1e9c9, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37573,1699781400943; forceNewPlan=false, retain=false 2023-11-12T09:30:04,204 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=daff3e7fcafd33ff24024100cbc1e9c9, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:04,211 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure daff3e7fcafd33ff24024100cbc1e9c9, server=jenkins-hbase3.apache.org,37573,1699781400943}] 2023-11-12T09:30:04,365 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:04,372 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1699781403532.daff3e7fcafd33ff24024100cbc1e9c9. 2023-11-12T09:30:04,372 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => daff3e7fcafd33ff24024100cbc1e9c9, NAME => 'hbase:namespace,,1699781403532.daff3e7fcafd33ff24024100cbc1e9c9.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:30:04,373 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:30:04,373 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace daff3e7fcafd33ff24024100cbc1e9c9 2023-11-12T09:30:04,374 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1699781403532.daff3e7fcafd33ff24024100cbc1e9c9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:30:04,374 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for daff3e7fcafd33ff24024100cbc1e9c9 2023-11-12T09:30:04,374 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for daff3e7fcafd33ff24024100cbc1e9c9 2023-11-12T09:30:04,376 INFO [StoreOpener-daff3e7fcafd33ff24024100cbc1e9c9-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region daff3e7fcafd33ff24024100cbc1e9c9 2023-11-12T09:30:04,379 INFO [StoreOpener-daff3e7fcafd33ff24024100cbc1e9c9-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region daff3e7fcafd33ff24024100cbc1e9c9 columnFamilyName info 2023-11-12T09:30:04,379 DEBUG [StoreOpener-daff3e7fcafd33ff24024100cbc1e9c9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:04,380 INFO [StoreOpener-daff3e7fcafd33ff24024100cbc1e9c9-1 {}] regionserver.HStore(324): Store=daff3e7fcafd33ff24024100cbc1e9c9/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:30:04,381 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/hbase/namespace/daff3e7fcafd33ff24024100cbc1e9c9 2023-11-12T09:30:04,382 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/hbase/namespace/daff3e7fcafd33ff24024100cbc1e9c9 2023-11-12T09:30:04,386 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for daff3e7fcafd33ff24024100cbc1e9c9 2023-11-12T09:30:04,390 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/hbase/namespace/daff3e7fcafd33ff24024100cbc1e9c9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:30:04,392 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened daff3e7fcafd33ff24024100cbc1e9c9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11820455200, jitterRate=0.1008656769990921}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:30:04,393 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for daff3e7fcafd33ff24024100cbc1e9c9: 2023-11-12T09:30:04,395 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:namespace,,1699781403532.daff3e7fcafd33ff24024100cbc1e9c9., pid=6, masterSystemTime=1699781404365 2023-11-12T09:30:04,399 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:namespace,,1699781403532.daff3e7fcafd33ff24024100cbc1e9c9. 2023-11-12T09:30:04,399 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1699781403532.daff3e7fcafd33ff24024100cbc1e9c9. 2023-11-12T09:30:04,400 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=daff3e7fcafd33ff24024100cbc1e9c9, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:04,408 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=6, resume processing ppid=5 2023-11-12T09:30:04,408 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure daff3e7fcafd33ff24024100cbc1e9c9, server=jenkins-hbase3.apache.org,37573,1699781400943 in 193 msec 2023-11-12T09:30:04,411 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=5, resume processing ppid=4 2023-11-12T09:30:04,411 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=daff3e7fcafd33ff24024100cbc1e9c9, ASSIGN in 359 msec 2023-11-12T09:30:04,412 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:30:04,412 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781404412"}]},"ts":"1699781404412"} 2023-11-12T09:30:04,415 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-11-12T09:30:04,419 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:30:04,422 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 886 msec 2023-11-12T09:30:04,446 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Set watcher on znode that does not yet exist, /1/namespace 2023-11-12T09:30:04,448 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/namespace 2023-11-12T09:30:04,448 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:30:04,448 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:30:04,504 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-11-12T09:30:04,524 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:30:04,531 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 36 msec 2023-11-12T09:30:04,539 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-11-12T09:30:04,553 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:30:04,558 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 18 msec 2023-11-12T09:30:04,581 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/default 2023-11-12T09:30:04,585 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/hbase 2023-11-12T09:30:04,586 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1194): Master has completed initialization 3.486sec 2023-11-12T09:30:04,590 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-11-12T09:30:04,592 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-11-12T09:30:04,594 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-11-12T09:30:04,595 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-11-12T09:30:04,595 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-11-12T09:30:04,598 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,44289,1699781397811-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-11-12T09:30:04,598 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,44289,1699781397811-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-11-12T09:30:04,619 DEBUG [Listener at localhost/33247 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x196df6a0 to 127.0.0.1:57951 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:30:04,627 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1297): Balancer post startup initialization complete, took 0 seconds 2023-11-12T09:30:04,628 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-11-12T09:30:04,628 DEBUG [Listener at localhost/33247 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c5c1226, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:30:04,633 DEBUG [Listener at localhost/33247 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-11-12T09:30:04,634 DEBUG [Listener at localhost/33247 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-11-12T09:30:04,649 DEBUG [hconnection-0x65d9fcc2-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:30:04,664 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:54270, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:30:04,679 INFO [Listener at localhost/33247 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,44289,1699781397811 2023-11-12T09:30:04,697 INFO [Listener at localhost/33247 {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/hadoop.log.dir so I do NOT create it in target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303 2023-11-12T09:30:04,698 INFO [Listener at localhost/33247 {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/hadoop.tmp.dir so I do NOT create it in target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303 2023-11-12T09:30:04,698 DEBUG [Listener at localhost/33247 {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303 2023-11-12T09:30:04,698 INFO [Listener at localhost/33247 {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/cluster_737aeac4-7599-9d23-aa4c-06ede0824ece, deleteOnExit=true 2023-11-12T09:30:04,698 INFO [Listener at localhost/33247 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/test.cache.data in system properties and HBase conf 2023-11-12T09:30:04,698 INFO [Listener at localhost/33247 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/hadoop.tmp.dir in system properties and HBase conf 2023-11-12T09:30:04,699 INFO [Listener at localhost/33247 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/hadoop.log.dir in system properties and HBase conf 2023-11-12T09:30:04,699 INFO [Listener at localhost/33247 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/mapreduce.cluster.local.dir in system properties and HBase conf 2023-11-12T09:30:04,699 INFO [Listener at localhost/33247 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-11-12T09:30:04,699 INFO [Listener at localhost/33247 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-11-12T09:30:04,699 DEBUG [Listener at localhost/33247 {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-11-12T09:30:04,700 INFO [Listener at localhost/33247 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-11-12T09:30:04,700 INFO [Listener at localhost/33247 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-11-12T09:30:04,700 INFO [Listener at localhost/33247 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-11-12T09:30:04,700 INFO [Listener at localhost/33247 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-12T09:30:04,701 INFO [Listener at localhost/33247 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-11-12T09:30:04,701 INFO [Listener at localhost/33247 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-11-12T09:30:04,701 INFO [Listener at localhost/33247 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-12T09:30:04,701 INFO [Listener at localhost/33247 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-12T09:30:04,701 INFO [Listener at localhost/33247 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-11-12T09:30:04,702 INFO [Listener at localhost/33247 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/nfs.dump.dir in system properties and HBase conf 2023-11-12T09:30:04,702 INFO [Listener at localhost/33247 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/java.io.tmpdir in system properties and HBase conf 2023-11-12T09:30:04,702 INFO [Listener at localhost/33247 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-12T09:30:04,702 INFO [Listener at localhost/33247 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-11-12T09:30:04,702 INFO [Listener at localhost/33247 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-11-12T09:30:04,721 WARN [Listener at localhost/33247 {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-11-12T09:30:04,721 WARN [Listener at localhost/33247 {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-11-12T09:30:04,780 WARN [Listener at localhost/33247 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:30:04,784 INFO [Listener at localhost/33247 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:30:04,793 INFO [Listener at localhost/33247 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/java.io.tmpdir/Jetty_localhost_40231_hdfs____gi8rlq/webapp 2023-11-12T09:30:04,895 INFO [Listener at localhost/33247 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:40231 2023-11-12T09:30:04,909 WARN [Listener at localhost/33247 {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-11-12T09:30:04,909 WARN [Listener at localhost/33247 {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-11-12T09:30:04,969 WARN [Listener at localhost/41663 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-11-12T09:30:05,005 WARN [Listener at localhost/41663 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-11-12T09:30:05,008 WARN [Listener at localhost/41663 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:30:05,009 INFO [Listener at localhost/41663 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:30:05,015 INFO [Listener at localhost/41663 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/java.io.tmpdir/Jetty_localhost_42849_datanode____.ngsyl6/webapp 2023-11-12T09:30:05,134 INFO [Listener at localhost/41663 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:42849 2023-11-12T09:30:05,170 WARN [Listener at localhost/46181 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-11-12T09:30:05,194 WARN [Listener at localhost/46181 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-11-12T09:30:05,198 WARN [Listener at localhost/46181 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:30:05,199 INFO [Listener at localhost/46181 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:30:05,205 INFO [Listener at localhost/46181 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/java.io.tmpdir/Jetty_localhost_40989_datanode____.f3bcwr/webapp 2023-11-12T09:30:05,281 WARN [Thread-202 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-11-12T09:30:05,336 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xdeb85177f6489b91: Processing first storage report for DS-76602b46-9427-4c0b-b54a-df5668fe8979 from datanode 47f26c7d-43a3-46f0-91e1-438554372551 2023-11-12T09:30:05,336 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xdeb85177f6489b91: from storage DS-76602b46-9427-4c0b-b54a-df5668fe8979 node DatanodeRegistration(127.0.0.1:33753, datanodeUuid=47f26c7d-43a3-46f0-91e1-438554372551, infoPort=36773, infoSecurePort=0, ipcPort=46181, storageInfo=lv=-57;cid=testClusterID;nsid=1754327384;c=1699781404724), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2023-11-12T09:30:05,337 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xdeb85177f6489b91: Processing first storage report for DS-4b20bd26-fb79-4084-bc1d-7cbeb78bc11a from datanode 47f26c7d-43a3-46f0-91e1-438554372551 2023-11-12T09:30:05,337 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xdeb85177f6489b91: from storage DS-4b20bd26-fb79-4084-bc1d-7cbeb78bc11a node DatanodeRegistration(127.0.0.1:33753, datanodeUuid=47f26c7d-43a3-46f0-91e1-438554372551, infoPort=36773, infoSecurePort=0, ipcPort=46181, storageInfo=lv=-57;cid=testClusterID;nsid=1754327384;c=1699781404724), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-11-12T09:30:05,358 INFO [Listener at localhost/46181 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:40989 2023-11-12T09:30:05,370 WARN [Listener at localhost/33833 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-11-12T09:30:05,404 WARN [Listener at localhost/33833 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-11-12T09:30:05,410 WARN [Listener at localhost/33833 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:30:05,411 INFO [Listener at localhost/33833 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:30:05,420 INFO [Listener at localhost/33833 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/java.io.tmpdir/Jetty_localhost_37253_datanode____.3t7xd9/webapp 2023-11-12T09:30:05,458 WARN [Thread-237 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-11-12T09:30:05,532 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xe503821d23880247: Processing first storage report for DS-970be131-3bb9-4ba7-9396-16778370691a from datanode 9976839f-8359-402e-861c-6a253624354b 2023-11-12T09:30:05,532 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xe503821d23880247: from storage DS-970be131-3bb9-4ba7-9396-16778370691a node DatanodeRegistration(127.0.0.1:41081, datanodeUuid=9976839f-8359-402e-861c-6a253624354b, infoPort=42245, infoSecurePort=0, ipcPort=33833, storageInfo=lv=-57;cid=testClusterID;nsid=1754327384;c=1699781404724), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2023-11-12T09:30:05,532 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xe503821d23880247: Processing first storage report for DS-573a2554-acff-4a63-b8bd-69748761a19b from datanode 9976839f-8359-402e-861c-6a253624354b 2023-11-12T09:30:05,532 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xe503821d23880247: from storage DS-573a2554-acff-4a63-b8bd-69748761a19b node DatanodeRegistration(127.0.0.1:41081, datanodeUuid=9976839f-8359-402e-861c-6a253624354b, infoPort=42245, infoSecurePort=0, ipcPort=33833, storageInfo=lv=-57;cid=testClusterID;nsid=1754327384;c=1699781404724), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-11-12T09:30:05,562 INFO [Listener at localhost/33833 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:37253 2023-11-12T09:30:05,582 WARN [Listener at localhost/36745 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-11-12T09:30:05,687 WARN [Thread-271 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-11-12T09:30:05,719 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x39560c52538f2c84: Processing first storage report for DS-d229501e-d2d8-4803-9b6a-369513c9bede from datanode a8473d81-fbfe-47b9-a243-4f97ea75cc63 2023-11-12T09:30:05,719 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x39560c52538f2c84: from storage DS-d229501e-d2d8-4803-9b6a-369513c9bede node DatanodeRegistration(127.0.0.1:39241, datanodeUuid=a8473d81-fbfe-47b9-a243-4f97ea75cc63, infoPort=38255, infoSecurePort=0, ipcPort=36745, storageInfo=lv=-57;cid=testClusterID;nsid=1754327384;c=1699781404724), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2023-11-12T09:30:05,719 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x39560c52538f2c84: Processing first storage report for DS-d699469e-2e89-455b-af22-56eff03192e8 from datanode a8473d81-fbfe-47b9-a243-4f97ea75cc63 2023-11-12T09:30:05,719 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x39560c52538f2c84: from storage DS-d699469e-2e89-455b-af22-56eff03192e8 node DatanodeRegistration(127.0.0.1:39241, datanodeUuid=a8473d81-fbfe-47b9-a243-4f97ea75cc63, infoPort=38255, infoSecurePort=0, ipcPort=36745, storageInfo=lv=-57;cid=testClusterID;nsid=1754327384;c=1699781404724), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-11-12T09:30:05,803 DEBUG [Listener at localhost/36745 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303 2023-11-12T09:30:05,803 INFO [Listener at localhost/36745 {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-11-12T09:30:05,803 INFO [Listener at localhost/36745 {}] hbase.HBaseTestingUtility(1143): NOT STARTING DFS 2023-11-12T09:30:05,803 INFO [Listener at localhost/36745 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:30:05,805 INFO [Listener at localhost/36745 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:30:05,830 INFO [Listener at localhost/36745 {}] util.FSUtils(462): Created version file at hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1 with version=8 2023-11-12T09:30:05,831 INFO [Listener at localhost/36745 {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/hbase-staging 2023-11-12T09:30:05,832 DEBUG [Listener at localhost/36745 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-11-12T09:30:05,832 DEBUG [Listener at localhost/36745 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-11-12T09:30:05,832 DEBUG [Listener at localhost/36745 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-11-12T09:30:05,833 DEBUG [Listener at localhost/36745 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-11-12T09:30:05,834 INFO [Listener at localhost/36745 {}] client.ConnectionUtils(128): master/jenkins-hbase3:0 server-side Connection retries=45 2023-11-12T09:30:05,834 INFO [Listener at localhost/36745 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-12T09:30:05,834 INFO [Listener at localhost/36745 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-11-12T09:30:05,834 INFO [Listener at localhost/36745 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-11-12T09:30:05,835 INFO [Listener at localhost/36745 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-12T09:30:05,835 INFO [Listener at localhost/36745 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-11-12T09:30:05,835 INFO [Listener at localhost/36745 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-11-12T09:30:05,835 INFO [Listener at localhost/36745 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-11-12T09:30:05,837 INFO [Listener at localhost/36745 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:37645 2023-11-12T09:30:05,838 INFO [Listener at localhost/36745 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:30:05,839 INFO [Listener at localhost/36745 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:30:05,842 INFO [Listener at localhost/36745 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:37645 connecting to ZooKeeper ensemble=127.0.0.1:57951 2023-11-12T09:30:05,846 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): master:376450x0, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-11-12T09:30:05,847 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:37645-0x1004cf7b7840004 connected 2023-11-12T09:30:05,863 DEBUG [Listener at localhost/36745 {}] zookeeper.ZKUtil(113): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Set watcher on znode that does not yet exist, /2/master 2023-11-12T09:30:05,864 DEBUG [Listener at localhost/36745 {}] zookeeper.ZKUtil(113): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Set watcher on znode that does not yet exist, /2/running 2023-11-12T09:30:05,864 DEBUG [Listener at localhost/36745 {}] zookeeper.ZKUtil(113): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Set watcher on znode that does not yet exist, /2/acl 2023-11-12T09:30:05,866 DEBUG [Listener at localhost/36745 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37645 2023-11-12T09:30:05,866 DEBUG [Listener at localhost/36745 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37645 2023-11-12T09:30:05,866 DEBUG [Listener at localhost/36745 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37645 2023-11-12T09:30:05,870 DEBUG [Listener at localhost/36745 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37645 2023-11-12T09:30:05,874 DEBUG [Listener at localhost/36745 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37645 2023-11-12T09:30:05,877 INFO [Listener at localhost/36745 {}] http.HttpServer(981): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-11-12T09:30:05,878 INFO [Listener at localhost/36745 {}] http.HttpServer(981): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-11-12T09:30:05,878 INFO [Listener at localhost/36745 {}] http.HttpServer(981): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-11-12T09:30:05,878 INFO [Listener at localhost/36745 {}] http.HttpServer(960): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-11-12T09:30:05,878 INFO [Listener at localhost/36745 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-11-12T09:30:05,878 INFO [Listener at localhost/36745 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-11-12T09:30:05,878 INFO [Listener at localhost/36745 {}] http.HttpServer(805): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-11-12T09:30:05,881 INFO [Listener at localhost/36745 {}] http.HttpServer(1227): Jetty bound to port 46331 2023-11-12T09:30:05,881 INFO [Listener at localhost/36745 {}] server.Server(375): jetty-9.4.52.v20230823; built: 2023-08-23T19:29:37.669Z; git: abdcda73818a1a2c705da276edb0bf6581e7997e; jvm 1.8.0_362-b09 2023-11-12T09:30:05,890 INFO [Listener at localhost/36745 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:30:05,890 INFO [Listener at localhost/36745 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@45b38ffb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/hadoop.log.dir/,AVAILABLE} 2023-11-12T09:30:05,891 INFO [Listener at localhost/36745 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:30:05,891 INFO [Listener at localhost/36745 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@6407dde3{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-11-12T09:30:05,906 INFO [Listener at localhost/36745 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-11-12T09:30:05,907 INFO [Listener at localhost/36745 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-11-12T09:30:05,907 INFO [Listener at localhost/36745 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-11-12T09:30:05,908 INFO [Listener at localhost/36745 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-11-12T09:30:05,909 INFO [Listener at localhost/36745 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:30:06,030 INFO [Listener at localhost/36745 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@28e08836{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-11-12T09:30:06,032 INFO [Listener at localhost/36745 {}] server.AbstractConnector(333): Started ServerConnector@10166c60{HTTP/1.1, (http/1.1)}{0.0.0.0:46331} 2023-11-12T09:30:06,033 INFO [Listener at localhost/36745 {}] server.Server(415): Started @13942ms 2023-11-12T09:30:06,033 INFO [Listener at localhost/36745 {}] master.HMaster(486): hbase.rootdir=hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1, hbase.cluster.distributed=false 2023-11-12T09:30:06,054 INFO [Listener at localhost/36745 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase3:0 server-side Connection retries=45 2023-11-12T09:30:06,054 INFO [Listener at localhost/36745 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-12T09:30:06,054 INFO [Listener at localhost/36745 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-11-12T09:30:06,054 INFO [Listener at localhost/36745 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-11-12T09:30:06,054 INFO [Listener at localhost/36745 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-12T09:30:06,055 INFO [Listener at localhost/36745 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-11-12T09:30:06,055 INFO [Listener at localhost/36745 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-11-12T09:30:06,055 INFO [Listener at localhost/36745 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-11-12T09:30:06,057 INFO [Listener at localhost/36745 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:33205 2023-11-12T09:30:06,058 INFO [Listener at localhost/36745 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-11-12T09:30:06,059 DEBUG [Listener at localhost/36745 {}] mob.MobFileCache(121): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-11-12T09:30:06,060 INFO [Listener at localhost/36745 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:30:06,062 INFO [Listener at localhost/36745 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:30:06,065 INFO [Listener at localhost/36745 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:33205 connecting to ZooKeeper ensemble=127.0.0.1:57951 2023-11-12T09:30:06,069 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): regionserver:332050x0, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-11-12T09:30:06,074 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:33205-0x1004cf7b7840005 connected 2023-11-12T09:30:06,074 DEBUG [Listener at localhost/36745 {}] zookeeper.ZKUtil(113): regionserver:33205-0x1004cf7b7840005, quorum=127.0.0.1:57951, baseZNode=/2 Set watcher on znode that does not yet exist, /2/master 2023-11-12T09:30:06,076 DEBUG [Listener at localhost/36745 {}] zookeeper.ZKUtil(113): regionserver:33205-0x1004cf7b7840005, quorum=127.0.0.1:57951, baseZNode=/2 Set watcher on znode that does not yet exist, /2/running 2023-11-12T09:30:06,076 DEBUG [Listener at localhost/36745 {}] zookeeper.ZKUtil(113): regionserver:33205-0x1004cf7b7840005, quorum=127.0.0.1:57951, baseZNode=/2 Set watcher on znode that does not yet exist, /2/acl 2023-11-12T09:30:06,077 DEBUG [Listener at localhost/36745 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33205 2023-11-12T09:30:06,078 DEBUG [Listener at localhost/36745 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33205 2023-11-12T09:30:06,085 DEBUG [Listener at localhost/36745 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33205 2023-11-12T09:30:06,086 DEBUG [Listener at localhost/36745 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33205 2023-11-12T09:30:06,087 DEBUG [Listener at localhost/36745 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33205 2023-11-12T09:30:06,094 INFO [Listener at localhost/36745 {}] http.HttpServer(981): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-11-12T09:30:06,094 INFO [Listener at localhost/36745 {}] http.HttpServer(981): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-11-12T09:30:06,094 INFO [Listener at localhost/36745 {}] http.HttpServer(981): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-11-12T09:30:06,095 INFO [Listener at localhost/36745 {}] http.HttpServer(960): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-11-12T09:30:06,095 INFO [Listener at localhost/36745 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-11-12T09:30:06,095 INFO [Listener at localhost/36745 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-11-12T09:30:06,096 INFO [Listener at localhost/36745 {}] http.HttpServer(805): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-11-12T09:30:06,097 INFO [Listener at localhost/36745 {}] http.HttpServer(1227): Jetty bound to port 39293 2023-11-12T09:30:06,097 INFO [Listener at localhost/36745 {}] server.Server(375): jetty-9.4.52.v20230823; built: 2023-08-23T19:29:37.669Z; git: abdcda73818a1a2c705da276edb0bf6581e7997e; jvm 1.8.0_362-b09 2023-11-12T09:30:06,102 INFO [Listener at localhost/36745 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:30:06,102 INFO [Listener at localhost/36745 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@1d942304{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/hadoop.log.dir/,AVAILABLE} 2023-11-12T09:30:06,103 INFO [Listener at localhost/36745 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:30:06,103 INFO [Listener at localhost/36745 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@588f8551{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-11-12T09:30:06,111 INFO [Listener at localhost/36745 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-11-12T09:30:06,112 INFO [Listener at localhost/36745 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-11-12T09:30:06,112 INFO [Listener at localhost/36745 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-11-12T09:30:06,112 INFO [Listener at localhost/36745 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2023-11-12T09:30:06,114 INFO [Listener at localhost/36745 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:30:06,114 INFO [Listener at localhost/36745 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@5d4d826a{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-11-12T09:30:06,116 INFO [Listener at localhost/36745 {}] server.AbstractConnector(333): Started ServerConnector@4ee83247{HTTP/1.1, (http/1.1)}{0.0.0.0:39293} 2023-11-12T09:30:06,116 INFO [Listener at localhost/36745 {}] server.Server(415): Started @14025ms 2023-11-12T09:30:06,118 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.52.v20230823; built: 2023-08-23T19:29:37.669Z; git: abdcda73818a1a2c705da276edb0bf6581e7997e; jvm 1.8.0_362-b09 2023-11-12T09:30:06,125 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@76f62d72{HTTP/1.1, (http/1.1)}{0.0.0.0:40025} 2023-11-12T09:30:06,125 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(415): Started @14034ms 2023-11-12T09:30:06,125 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2389): Adding backup master ZNode /2/backup-masters/jenkins-hbase3.apache.org,37645,1699781405833 2023-11-12T09:30:06,129 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/backup-masters 2023-11-12T09:30:06,129 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33205-0x1004cf7b7840005, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/backup-masters 2023-11-12T09:30:06,130 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Set watcher on existing znode=/2/backup-masters/jenkins-hbase3.apache.org,37645,1699781405833 2023-11-12T09:30:06,131 DEBUG [M:0;jenkins-hbase3:37645 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:37645 2023-11-12T09:30:06,132 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33205-0x1004cf7b7840005, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/2/master 2023-11-12T09:30:06,132 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33205-0x1004cf7b7840005, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-12T09:30:06,132 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/2/master 2023-11-12T09:30:06,132 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-12T09:30:06,133 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Set watcher on existing znode=/2/master 2023-11-12T09:30:06,134 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /2/backup-masters/jenkins-hbase3.apache.org,37645,1699781405833 from backup master directory 2023-11-12T09:30:06,138 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/2/backup-masters/jenkins-hbase3.apache.org,37645,1699781405833 2023-11-12T09:30:06,138 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/backup-masters 2023-11-12T09:30:06,138 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33205-0x1004cf7b7840005, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/backup-masters 2023-11-12T09:30:06,138 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Set watcher on existing znode=/2/master 2023-11-12T09:30:06,138 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-11-12T09:30:06,138 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,37645,1699781405833 2023-11-12T09:30:06,167 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(611): Created cluster ID file at hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/hbase.id with ID: 18962475-6e98-44af-8e90-598789639d3d 2023-11-12T09:30:06,193 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:30:06,197 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-12T09:30:06,197 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33205-0x1004cf7b7840005, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-12T09:30:06,624 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:30:06,625 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-11-12T09:30:06,626 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-12T09:30:07,059 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/MasterData/data/master/store 2023-11-12T09:30:07,078 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:30:07,078 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-11-12T09:30:07,079 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:30:07,079 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:30:07,079 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-11-12T09:30:07,079 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:30:07,079 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:30:07,079 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-11-12T09:30:07,084 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/MasterData/data/master/store/.initializing 2023-11-12T09:30:07,084 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:41663/tmp/wal/MasterData/WALs/jenkins-hbase3.apache.org,37645,1699781405833 2023-11-12T09:30:07,086 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-11-12T09:30:07,089 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C37645%2C1699781405833, suffix=, logDir=hdfs://localhost:41663/tmp/wal/MasterData/WALs/jenkins-hbase3.apache.org,37645,1699781405833, archiveDir=hdfs://localhost:41663/tmp/wal/MasterData/oldWALs, maxLogs=10 2023-11-12T09:30:07,106 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /tmp/wal/MasterData/WALs/jenkins-hbase3.apache.org,37645,1699781405833/jenkins-hbase3.apache.org%2C37645%2C1699781405833.1699781407089, exclude list is [], retry=0 2023-11-12T09:30:07,111 DEBUG [RS-EventLoopGroup-6-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39241,DS-d229501e-d2d8-4803-9b6a-369513c9bede,DISK] 2023-11-12T09:30:07,115 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /tmp/wal/MasterData/WALs/jenkins-hbase3.apache.org,37645,1699781405833/jenkins-hbase3.apache.org%2C37645%2C1699781405833.1699781407089 2023-11-12T09:30:07,115 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:39241,DS-d229501e-d2d8-4803-9b6a-369513c9bede,DISK]] 2023-11-12T09:30:07,115 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:30:07,116 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:30:07,116 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:30:07,116 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:30:07,121 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:30:07,123 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-11-12T09:30:07,123 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:07,124 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:30:07,124 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:30:07,127 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-11-12T09:30:07,127 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:07,128 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:30:07,128 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:30:07,131 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-11-12T09:30:07,131 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:07,132 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:30:07,132 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:30:07,134 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-11-12T09:30:07,134 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:07,135 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:30:07,136 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:30:07,137 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:30:07,137 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41663/tmp/wal/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:30:07,141 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-11-12T09:30:07,143 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:30:07,146 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41663/tmp/wal/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:30:07,147 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10811152000, jitterRate=0.0068669915199279785}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-11-12T09:30:07,147 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-11-12T09:30:07,148 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-11-12T09:30:07,152 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ba26c62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:30:07,153 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(861): No meta location available on zookeeper, skip migrating... 2023-11-12T09:30:07,154 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-11-12T09:30:07,154 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(563): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-11-12T09:30:07,154 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-11-12T09:30:07,155 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(583): Recovered RegionProcedureStore lease in 0 msec 2023-11-12T09:30:07,155 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(597): Loaded RegionProcedureStore in 0 msec 2023-11-12T09:30:07,155 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-11-12T09:30:07,157 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-11-12T09:30:07,158 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Unable to get data of znode /2/balancer because node does not exist (not necessarily an error) 2023-11-12T09:30:07,160 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /2/balancer already deleted, retry=false 2023-11-12T09:30:07,160 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-11-12T09:30:07,161 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Unable to get data of znode /2/normalizer because node does not exist (not necessarily an error) 2023-11-12T09:30:07,162 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /2/normalizer already deleted, retry=false 2023-11-12T09:30:07,162 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-11-12T09:30:07,163 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Unable to get data of znode /2/switch/split because node does not exist (not necessarily an error) 2023-11-12T09:30:07,167 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /2/switch/split already deleted, retry=false 2023-11-12T09:30:07,168 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Unable to get data of znode /2/switch/merge because node does not exist (not necessarily an error) 2023-11-12T09:30:07,169 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /2/switch/merge already deleted, retry=false 2023-11-12T09:30:07,171 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Unable to get data of znode /2/snapshot-cleanup because node does not exist (not necessarily an error) 2023-11-12T09:30:07,172 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /2/snapshot-cleanup already deleted, retry=false 2023-11-12T09:30:07,177 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33205-0x1004cf7b7840005, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/2/running 2023-11-12T09:30:07,177 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/2/running 2023-11-12T09:30:07,177 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33205-0x1004cf7b7840005, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-12T09:30:07,177 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-12T09:30:07,178 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(805): Active/primary master=jenkins-hbase3.apache.org,37645,1699781405833, sessionid=0x1004cf7b7840004, setting cluster-up flag (Was=false) 2023-11-12T09:30:07,178 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-11-12T09:30:07,181 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-12T09:30:07,182 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33205-0x1004cf7b7840005, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-12T09:30:07,190 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /2/flush-table-proc/acquired, /2/flush-table-proc/reached, /2/flush-table-proc/abort 2023-11-12T09:30:07,192 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,37645,1699781405833 2023-11-12T09:30:07,194 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33205-0x1004cf7b7840005, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-12T09:30:07,194 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-12T09:30:07,202 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /2/rolllog-proc/acquired, /2/rolllog-proc/reached, /2/rolllog-proc/abort 2023-11-12T09:30:07,203 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,37645,1699781405833 2023-11-12T09:30:07,207 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-12T09:30:07,207 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33205-0x1004cf7b7840005, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-12T09:30:07,213 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /2/online-snapshot/acquired, /2/online-snapshot/reached, /2/online-snapshot/abort 2023-11-12T09:30:07,214 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,37645,1699781405833 2023-11-12T09:30:07,218 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-11-12T09:30:07,219 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-11-12T09:30:07,219 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-11-12T09:30:07,228 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,37645,1699781405833 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-11-12T09:30:07,229 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-11-12T09:30:07,229 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-11-12T09:30:07,229 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-11-12T09:30:07,229 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-11-12T09:30:07,229 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-11-12T09:30:07,229 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:30:07,229 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-11-12T09:30:07,230 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:30:07,231 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1699781437231 2023-11-12T09:30:07,231 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-11-12T09:30:07,231 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-11-12T09:30:07,231 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-11-12T09:30:07,231 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-11-12T09:30:07,232 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-11-12T09:30:07,232 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-11-12T09:30:07,232 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-11-12T09:30:07,234 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-11-12T09:30:07,234 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-11-12T09:30:07,234 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-11-12T09:30:07,236 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:07,236 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-11-12T09:30:07,237 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-11-12T09:30:07,238 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-11-12T09:30:07,238 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-11-12T09:30:07,241 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x43dfa29b to 127.0.0.1:57951 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:30:07,241 DEBUG [RS:0;jenkins-hbase3:33205 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:33205 2023-11-12T09:30:07,252 INFO [RS:0;jenkins-hbase3:33205 {}] regionserver.HRegionServer(997): ClusterId : 18962475-6e98-44af-8e90-598789639d3d 2023-11-12T09:30:07,253 INFO [RS:0;jenkins-hbase3:33205 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-11-12T09:30:07,253 DEBUG [RS:0;jenkins-hbase3:33205 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-11-12T09:30:07,264 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@da12fae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:30:07,264 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-11-12T09:30:07,265 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-11-12T09:30:07,265 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-11-12T09:30:07,265 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-11-12T09:30:07,265 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1699781407265,5,FailOnTimeoutGroup] 2023-11-12T09:30:07,265 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1 2023-11-12T09:30:07,265 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1699781407265,5,FailOnTimeoutGroup] 2023-11-12T09:30:07,265 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-11-12T09:30:07,265 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1636): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-11-12T09:30:07,265 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-11-12T09:30:07,266 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-11-12T09:30:07,266 DEBUG [RS:0;jenkins-hbase3:33205 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-11-12T09:30:07,266 DEBUG [RS:0;jenkins-hbase3:33205 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-11-12T09:30:07,269 DEBUG [RS:0;jenkins-hbase3:33205 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-11-12T09:30:07,269 DEBUG [RS:0;jenkins-hbase3:33205 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-11-12T09:30:07,282 DEBUG [RS:0;jenkins-hbase3:33205 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-11-12T09:30:07,287 DEBUG [RS:0;jenkins-hbase3:33205 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@713f0f56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:30:07,297 DEBUG [RS:0;jenkins-hbase3:33205 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fa089fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-11-12T09:30:07,297 INFO [RS:0;jenkins-hbase3:33205 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-11-12T09:30:07,297 INFO [RS:0;jenkins-hbase3:33205 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-11-12T09:30:07,298 DEBUG [RS:0;jenkins-hbase3:33205 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-11-12T09:30:07,302 INFO [RS:0;jenkins-hbase3:33205 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,37645,1699781405833 with isa=jenkins-hbase3.apache.org/172.31.12.81:33205, startcode=1699781406053 2023-11-12T09:30:07,302 DEBUG [RS:0;jenkins-hbase3:33205 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-11-12T09:30:07,308 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:30:07,308 INFO [RS-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:48811, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2023-11-12T09:30:07,314 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37645 {}] master.ServerManager(396): Registering regionserver=jenkins-hbase3.apache.org,33205,1699781406053 2023-11-12T09:30:07,314 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-11-12T09:30:07,319 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-11-12T09:30:07,319 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:07,320 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:30:07,320 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-11-12T09:30:07,322 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-11-12T09:30:07,322 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:07,323 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:30:07,323 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-11-12T09:30:07,325 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-11-12T09:30:07,326 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:07,326 DEBUG [RS:0;jenkins-hbase3:33205 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1 2023-11-12T09:30:07,326 DEBUG [RS:0;jenkins-hbase3:33205 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:41663 2023-11-12T09:30:07,326 DEBUG [RS:0;jenkins-hbase3:33205 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=46331 2023-11-12T09:30:07,326 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:30:07,328 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/data/hbase/meta/1588230740 2023-11-12T09:30:07,329 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/data/hbase/meta/1588230740 2023-11-12T09:30:07,330 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41663/tmp/wal/data/hbase/meta/1588230740 2023-11-12T09:30:07,331 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/rs 2023-11-12T09:30:07,332 DEBUG [RS:0;jenkins-hbase3:33205 {}] zookeeper.ZKUtil(111): regionserver:33205-0x1004cf7b7840005, quorum=127.0.0.1:57951, baseZNode=/2 Set watcher on existing znode=/2/rs/jenkins-hbase3.apache.org,33205,1699781406053 2023-11-12T09:30:07,332 WARN [RS:0;jenkins-hbase3:33205 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-11-12T09:30:07,332 INFO [RS:0;jenkins-hbase3:33205 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-12T09:30:07,332 DEBUG [RS:0;jenkins-hbase3:33205 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:41663/tmp/wal/WALs/jenkins-hbase3.apache.org,33205,1699781406053 2023-11-12T09:30:07,334 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-11-12T09:30:07,336 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-11-12T09:30:07,343 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,33205,1699781406053] 2023-11-12T09:30:07,350 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41663/tmp/wal/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:30:07,353 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9570510560, jitterRate=-0.10867674648761749}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-11-12T09:30:07,353 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-11-12T09:30:07,354 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-11-12T09:30:07,354 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-11-12T09:30:07,354 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-11-12T09:30:07,354 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-11-12T09:30:07,354 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-11-12T09:30:07,359 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-11-12T09:30:07,359 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-11-12T09:30:07,361 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-11-12T09:30:07,361 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-11-12T09:30:07,361 DEBUG [RS:0;jenkins-hbase3:33205 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-11-12T09:30:07,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-11-12T09:30:07,362 INFO [RS:0;jenkins-hbase3:33205 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-11-12T09:30:07,368 INFO [RS:0;jenkins-hbase3:33205 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-11-12T09:30:07,368 INFO [RS:0;jenkins-hbase3:33205 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-11-12T09:30:07,368 INFO [RS:0;jenkins-hbase3:33205 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-11-12T09:30:07,379 INFO [RS:0;jenkins-hbase3:33205 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-11-12T09:30:07,379 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-11-12T09:30:07,382 INFO [RS:0;jenkins-hbase3:33205 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-11-12T09:30:07,382 DEBUG [RS:0;jenkins-hbase3:33205 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:30:07,382 DEBUG [RS:0;jenkins-hbase3:33205 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:30:07,383 DEBUG [RS:0;jenkins-hbase3:33205 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:30:07,383 DEBUG [RS:0;jenkins-hbase3:33205 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:30:07,383 DEBUG [RS:0;jenkins-hbase3:33205 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:30:07,383 DEBUG [RS:0;jenkins-hbase3:33205 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-11-12T09:30:07,383 DEBUG [RS:0;jenkins-hbase3:33205 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:30:07,383 DEBUG [RS:0;jenkins-hbase3:33205 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:30:07,383 DEBUG [RS:0;jenkins-hbase3:33205 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:30:07,383 DEBUG [RS:0;jenkins-hbase3:33205 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:30:07,384 DEBUG [RS:0;jenkins-hbase3:33205 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:30:07,384 DEBUG [RS:0;jenkins-hbase3:33205 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-11-12T09:30:07,384 DEBUG [RS:0;jenkins-hbase3:33205 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-11-12T09:30:07,384 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-11-12T09:30:07,390 INFO [RS:0;jenkins-hbase3:33205 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-11-12T09:30:07,390 INFO [RS:0;jenkins-hbase3:33205 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-11-12T09:30:07,390 INFO [RS:0;jenkins-hbase3:33205 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-11-12T09:30:07,390 INFO [RS:0;jenkins-hbase3:33205 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-11-12T09:30:07,390 INFO [RS:0;jenkins-hbase3:33205 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,33205,1699781406053-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-11-12T09:30:07,409 INFO [RS:0;jenkins-hbase3:33205 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-11-12T09:30:07,409 INFO [RS:0;jenkins-hbase3:33205 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,33205,1699781406053-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-11-12T09:30:07,426 INFO [RS:0;jenkins-hbase3:33205 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,33205,1699781406053 started 2023-11-12T09:30:07,426 INFO [RS:0;jenkins-hbase3:33205 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,33205,1699781406053, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:33205, sessionid=0x1004cf7b7840005 2023-11-12T09:30:07,426 DEBUG [RS:0;jenkins-hbase3:33205 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-11-12T09:30:07,426 DEBUG [RS:0;jenkins-hbase3:33205 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager jenkins-hbase3.apache.org,33205,1699781406053 2023-11-12T09:30:07,426 DEBUG [RS:0;jenkins-hbase3:33205 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,33205,1699781406053' 2023-11-12T09:30:07,426 DEBUG [RS:0;jenkins-hbase3:33205 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/2/flush-table-proc/abort' 2023-11-12T09:30:07,427 DEBUG [RS:0;jenkins-hbase3:33205 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/2/flush-table-proc/acquired' 2023-11-12T09:30:07,428 DEBUG [RS:0;jenkins-hbase3:33205 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-11-12T09:30:07,428 DEBUG [RS:0;jenkins-hbase3:33205 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-11-12T09:30:07,428 DEBUG [RS:0;jenkins-hbase3:33205 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,33205,1699781406053' 2023-11-12T09:30:07,428 DEBUG [RS:0;jenkins-hbase3:33205 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/2/rolllog-proc/abort' 2023-11-12T09:30:07,434 DEBUG [RS:0;jenkins-hbase3:33205 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/2/rolllog-proc/acquired' 2023-11-12T09:30:07,434 INFO [RS:0;jenkins-hbase3:33205 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-11-12T09:30:07,434 DEBUG [RS:0;jenkins-hbase3:33205 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-11-12T09:30:07,434 DEBUG [RS:0;jenkins-hbase3:33205 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-11-12T09:30:07,434 DEBUG [RS:0;jenkins-hbase3:33205 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,33205,1699781406053 2023-11-12T09:30:07,434 DEBUG [RS:0;jenkins-hbase3:33205 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,33205,1699781406053' 2023-11-12T09:30:07,434 DEBUG [RS:0;jenkins-hbase3:33205 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/2/online-snapshot/abort' 2023-11-12T09:30:07,435 DEBUG [RS:0;jenkins-hbase3:33205 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/2/online-snapshot/acquired' 2023-11-12T09:30:07,439 DEBUG [RS:0;jenkins-hbase3:33205 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-11-12T09:30:07,439 INFO [RS:0;jenkins-hbase3:33205 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-11-12T09:30:07,439 INFO [RS:0;jenkins-hbase3:33205 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-11-12T09:30:07,536 WARN [jenkins-hbase3:37645 {}] assignment.AssignmentManager(2298): No servers available; cannot place 1 unassigned regions. 2023-11-12T09:30:07,540 INFO [RS:0;jenkins-hbase3:33205 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-11-12T09:30:07,543 INFO [RS:0;jenkins-hbase3:33205 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C33205%2C1699781406053, suffix=, logDir=hdfs://localhost:41663/tmp/wal/WALs/jenkins-hbase3.apache.org,33205,1699781406053, archiveDir=hdfs://localhost:41663/tmp/wal/oldWALs, maxLogs=32 2023-11-12T09:30:07,560 DEBUG [RS:0;jenkins-hbase3:33205 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /tmp/wal/WALs/jenkins-hbase3.apache.org,33205,1699781406053/jenkins-hbase3.apache.org%2C33205%2C1699781406053.1699781407545, exclude list is [], retry=0 2023-11-12T09:30:07,573 DEBUG [RS-EventLoopGroup-6-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39241,DS-d229501e-d2d8-4803-9b6a-369513c9bede,DISK] 2023-11-12T09:30:07,578 INFO [RS:0;jenkins-hbase3:33205 {}] wal.AbstractFSWAL(832): New WAL /tmp/wal/WALs/jenkins-hbase3.apache.org,33205,1699781406053/jenkins-hbase3.apache.org%2C33205%2C1699781406053.1699781407545 2023-11-12T09:30:07,578 DEBUG [RS:0;jenkins-hbase3:33205 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:39241,DS-d229501e-d2d8-4803-9b6a-369513c9bede,DISK]] 2023-11-12T09:30:07,786 DEBUG [jenkins-hbase3:37645 {}] assignment.AssignmentManager(2319): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-11-12T09:30:07,788 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,33205,1699781406053 2023-11-12T09:30:07,790 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,33205,1699781406053, state=OPENING 2023-11-12T09:30:07,792 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-11-12T09:30:07,794 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33205-0x1004cf7b7840005, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-12T09:30:07,794 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-12T09:30:07,794 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,33205,1699781406053}] 2023-11-12T09:30:07,794 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /2/meta-region-server: CHANGED 2023-11-12T09:30:07,794 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /2/meta-region-server: CHANGED 2023-11-12T09:30:07,953 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,33205,1699781406053 2023-11-12T09:30:07,953 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2023-11-12T09:30:07,957 INFO [RS-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:44958, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-11-12T09:30:07,963 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-11-12T09:30:07,963 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-12T09:30:07,963 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-11-12T09:30:07,966 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C33205%2C1699781406053.meta, suffix=.meta, logDir=hdfs://localhost:41663/tmp/wal/WALs/jenkins-hbase3.apache.org,33205,1699781406053, archiveDir=hdfs://localhost:41663/tmp/wal/oldWALs, maxLogs=32 2023-11-12T09:30:07,978 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /tmp/wal/WALs/jenkins-hbase3.apache.org,33205,1699781406053/jenkins-hbase3.apache.org%2C33205%2C1699781406053.meta.1699781407967.meta, exclude list is [], retry=0 2023-11-12T09:30:07,995 DEBUG [RS-EventLoopGroup-6-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41081,DS-970be131-3bb9-4ba7-9396-16778370691a,DISK] 2023-11-12T09:30:08,007 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /tmp/wal/WALs/jenkins-hbase3.apache.org,33205,1699781406053/jenkins-hbase3.apache.org%2C33205%2C1699781406053.meta.1699781407967.meta 2023-11-12T09:30:08,014 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41081,DS-970be131-3bb9-4ba7-9396-16778370691a,DISK]] 2023-11-12T09:30:08,015 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:30:08,015 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:30:08,015 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-11-12T09:30:08,015 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-11-12T09:30:08,015 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-11-12T09:30:08,016 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-11-12T09:30:08,016 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:30:08,016 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-11-12T09:30:08,016 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-11-12T09:30:08,023 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-11-12T09:30:08,024 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-11-12T09:30:08,024 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:08,025 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:30:08,025 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-11-12T09:30:08,026 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-11-12T09:30:08,026 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:08,027 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:30:08,027 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-11-12T09:30:08,029 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-11-12T09:30:08,029 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:08,029 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:30:08,031 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/data/hbase/meta/1588230740 2023-11-12T09:30:08,031 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/data/hbase/meta/1588230740 2023-11-12T09:30:08,033 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41663/tmp/wal/data/hbase/meta/1588230740 2023-11-12T09:30:08,036 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-11-12T09:30:08,039 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-11-12T09:30:08,040 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9997299680, jitterRate=-0.06892891228199005}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-11-12T09:30:08,040 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-11-12T09:30:08,043 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1699781407952 2023-11-12T09:30:08,050 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:meta,,1.1588230740 2023-11-12T09:30:08,050 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-11-12T09:30:08,051 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,33205,1699781406053 2023-11-12T09:30:08,053 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,33205,1699781406053, state=OPEN 2023-11-12T09:30:08,055 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33205-0x1004cf7b7840005, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/2/meta-region-server 2023-11-12T09:30:08,055 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/2/meta-region-server 2023-11-12T09:30:08,055 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /2/meta-region-server: CHANGED 2023-11-12T09:30:08,055 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /2/meta-region-server: CHANGED 2023-11-12T09:30:08,059 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=3, resume processing ppid=2 2023-11-12T09:30:08,059 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,33205,1699781406053 in 261 msec 2023-11-12T09:30:08,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=2, resume processing ppid=1 2023-11-12T09:30:08,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 698 msec 2023-11-12T09:30:08,065 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 847 msec 2023-11-12T09:30:08,065 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1064): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1699781408065, completionTime=-1 2023-11-12T09:30:08,065 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(816): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-11-12T09:30:08,065 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1645): Joining cluster... 2023-11-12T09:30:08,072 DEBUG [hconnection-0x55e15f11-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:30:08,076 INFO [RS-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:44960, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:30:08,078 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1657): Number of RegionServers=1 2023-11-12T09:30:08,078 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1699781468078 2023-11-12T09:30:08,078 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1699781528078 2023-11-12T09:30:08,078 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1664): Joined the cluster in 12 msec 2023-11-12T09:30:08,087 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37645,1699781405833-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-11-12T09:30:08,087 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37645,1699781405833-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-11-12T09:30:08,087 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37645,1699781405833-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-11-12T09:30:08,087 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:37645, period=300000, unit=MILLISECONDS is enabled. 2023-11-12T09:30:08,087 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-11-12T09:30:08,087 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-11-12T09:30:08,088 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2369): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-11-12T09:30:08,089 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-11-12T09:30:08,090 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-11-12T09:30:08,091 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:30:08,091 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:08,092 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:30:08,510 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 3e47234fadef92f1114692ec74b0ca02, NAME => 'hbase:namespace,,1699781408087.3e47234fadef92f1114692ec74b0ca02.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1 2023-11-12T09:30:08,530 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1699781408087.3e47234fadef92f1114692ec74b0ca02.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:30:08,530 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing 3e47234fadef92f1114692ec74b0ca02, disabling compactions & flushes 2023-11-12T09:30:08,530 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1699781408087.3e47234fadef92f1114692ec74b0ca02. 2023-11-12T09:30:08,531 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1699781408087.3e47234fadef92f1114692ec74b0ca02. 2023-11-12T09:30:08,531 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1699781408087.3e47234fadef92f1114692ec74b0ca02. after waiting 0 ms 2023-11-12T09:30:08,531 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1699781408087.3e47234fadef92f1114692ec74b0ca02. 2023-11-12T09:30:08,531 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1699781408087.3e47234fadef92f1114692ec74b0ca02. 2023-11-12T09:30:08,531 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for 3e47234fadef92f1114692ec74b0ca02: 2023-11-12T09:30:08,533 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:30:08,533 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1699781408087.3e47234fadef92f1114692ec74b0ca02.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1699781408533"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781408533"}]},"ts":"1699781408533"} 2023-11-12T09:30:08,537 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:30:08,538 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:30:08,538 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781408538"}]},"ts":"1699781408538"} 2023-11-12T09:30:08,541 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-11-12T09:30:08,546 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=3e47234fadef92f1114692ec74b0ca02, ASSIGN}] 2023-11-12T09:30:08,549 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=3e47234fadef92f1114692ec74b0ca02, ASSIGN 2023-11-12T09:30:08,551 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=3e47234fadef92f1114692ec74b0ca02, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,33205,1699781406053; forceNewPlan=false, retain=false 2023-11-12T09:30:08,703 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=3e47234fadef92f1114692ec74b0ca02, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,33205,1699781406053 2023-11-12T09:30:08,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 3e47234fadef92f1114692ec74b0ca02, server=jenkins-hbase3.apache.org,33205,1699781406053}] 2023-11-12T09:30:08,831 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2023-11-12T09:30:08,859 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,33205,1699781406053 2023-11-12T09:30:08,886 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1699781408087.3e47234fadef92f1114692ec74b0ca02. 2023-11-12T09:30:08,887 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => 3e47234fadef92f1114692ec74b0ca02, NAME => 'hbase:namespace,,1699781408087.3e47234fadef92f1114692ec74b0ca02.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:30:08,889 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:30:08,890 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 3e47234fadef92f1114692ec74b0ca02 2023-11-12T09:30:08,890 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1699781408087.3e47234fadef92f1114692ec74b0ca02.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:30:08,890 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for 3e47234fadef92f1114692ec74b0ca02 2023-11-12T09:30:08,890 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for 3e47234fadef92f1114692ec74b0ca02 2023-11-12T09:30:08,895 INFO [StoreOpener-3e47234fadef92f1114692ec74b0ca02-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 3e47234fadef92f1114692ec74b0ca02 2023-11-12T09:30:08,897 INFO [StoreOpener-3e47234fadef92f1114692ec74b0ca02-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3e47234fadef92f1114692ec74b0ca02 columnFamilyName info 2023-11-12T09:30:08,898 DEBUG [StoreOpener-3e47234fadef92f1114692ec74b0ca02-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:08,898 INFO [StoreOpener-3e47234fadef92f1114692ec74b0ca02-1 {}] regionserver.HStore(324): Store=3e47234fadef92f1114692ec74b0ca02/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:30:08,901 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/data/hbase/namespace/3e47234fadef92f1114692ec74b0ca02 2023-11-12T09:30:08,902 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/data/hbase/namespace/3e47234fadef92f1114692ec74b0ca02 2023-11-12T09:30:08,908 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41663/tmp/wal/data/hbase/namespace/3e47234fadef92f1114692ec74b0ca02 2023-11-12T09:30:08,919 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for 3e47234fadef92f1114692ec74b0ca02 2023-11-12T09:30:08,939 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41663/tmp/wal/data/hbase/namespace/3e47234fadef92f1114692ec74b0ca02/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:30:08,940 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened 3e47234fadef92f1114692ec74b0ca02; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11448793280, jitterRate=0.0662519633769989}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:30:08,940 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for 3e47234fadef92f1114692ec74b0ca02: 2023-11-12T09:30:08,942 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:namespace,,1699781408087.3e47234fadef92f1114692ec74b0ca02., pid=6, masterSystemTime=1699781408859 2023-11-12T09:30:08,946 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:namespace,,1699781408087.3e47234fadef92f1114692ec74b0ca02. 2023-11-12T09:30:08,946 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1699781408087.3e47234fadef92f1114692ec74b0ca02. 2023-11-12T09:30:08,947 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=3e47234fadef92f1114692ec74b0ca02, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,33205,1699781406053 2023-11-12T09:30:08,963 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=6, resume processing ppid=5 2023-11-12T09:30:08,963 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 3e47234fadef92f1114692ec74b0ca02, server=jenkins-hbase3.apache.org,33205,1699781406053 in 245 msec 2023-11-12T09:30:08,968 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=5, resume processing ppid=4 2023-11-12T09:30:08,968 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=3e47234fadef92f1114692ec74b0ca02, ASSIGN in 417 msec 2023-11-12T09:30:08,970 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:30:08,970 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781408970"}]},"ts":"1699781408970"} 2023-11-12T09:30:08,974 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-11-12T09:30:08,980 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:30:08,983 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 892 msec 2023-11-12T09:30:08,991 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Set watcher on znode that does not yet exist, /2/namespace 2023-11-12T09:30:08,993 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33205-0x1004cf7b7840005, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-12T09:30:08,993 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/2/namespace 2023-11-12T09:30:08,993 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-12T09:30:09,002 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-11-12T09:30:09,004 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-11-12T09:30:09,006 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-11-12T09:30:09,007 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-11-12T09:30:09,009 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-11-12T09:30:09,009 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-11-12T09:30:09,011 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:30:09,011 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2023-11-12T09:30:09,011 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-11-12T09:30:09,011 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-11-12T09:30:09,011 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-11-12T09:30:09,012 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-11-12T09:30:09,022 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-11-12T09:30:09,037 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/namespace 2023-11-12T09:30:09,045 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 21 msec 2023-11-12T09:30:09,055 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-11-12T09:30:09,067 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/namespace 2023-11-12T09:30:09,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 16 msec 2023-11-12T09:30:09,094 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/2/namespace/default 2023-11-12T09:30:09,096 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/2/namespace/hbase 2023-11-12T09:30:09,096 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1194): Master has completed initialization 2.958sec 2023-11-12T09:30:09,096 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-11-12T09:30:09,096 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-11-12T09:30:09,097 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-11-12T09:30:09,097 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-11-12T09:30:09,097 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-11-12T09:30:09,097 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37645,1699781405833-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-11-12T09:30:09,097 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37645,1699781405833-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-11-12T09:30:09,100 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1297): Balancer post startup initialization complete, took 0 seconds 2023-11-12T09:30:09,100 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-11-12T09:30:09,134 DEBUG [Listener at localhost/36745 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x66e0d3e8 to 127.0.0.1:57951 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:30:09,141 DEBUG [Listener at localhost/36745 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38869276, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:30:09,145 DEBUG [hconnection-0x797e2988-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:30:09,148 INFO [RS-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:44964, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:30:09,151 INFO [Listener at localhost/36745 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,37645,1699781405833 2023-11-12T09:30:09,151 INFO [Listener at localhost/36745 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-11-12T09:30:09,151 INFO [Listener at localhost/36745 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/test.cache.data in system properties and HBase conf 2023-11-12T09:30:09,151 INFO [Listener at localhost/36745 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/hadoop.tmp.dir in system properties and HBase conf 2023-11-12T09:30:09,151 INFO [Listener at localhost/36745 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/hadoop.log.dir in system properties and HBase conf 2023-11-12T09:30:09,151 INFO [Listener at localhost/36745 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/mapreduce.cluster.local.dir in system properties and HBase conf 2023-11-12T09:30:09,152 INFO [Listener at localhost/36745 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-11-12T09:30:09,152 INFO [Listener at localhost/36745 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-11-12T09:30:09,152 INFO [Listener at localhost/36745 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-11-12T09:30:09,152 INFO [Listener at localhost/36745 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-11-12T09:30:09,152 INFO [Listener at localhost/36745 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-11-12T09:30:09,152 INFO [Listener at localhost/36745 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-12T09:30:09,153 INFO [Listener at localhost/36745 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-11-12T09:30:09,153 INFO [Listener at localhost/36745 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-11-12T09:30:09,153 INFO [Listener at localhost/36745 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-12T09:30:09,153 INFO [Listener at localhost/36745 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-12T09:30:09,153 INFO [Listener at localhost/36745 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-11-12T09:30:09,153 INFO [Listener at localhost/36745 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/nfs.dump.dir in system properties and HBase conf 2023-11-12T09:30:09,153 INFO [Listener at localhost/36745 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/java.io.tmpdir in system properties and HBase conf 2023-11-12T09:30:09,153 INFO [Listener at localhost/36745 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-12T09:30:09,154 INFO [Listener at localhost/36745 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-11-12T09:30:09,154 INFO [Listener at localhost/36745 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-11-12T09:30:11,168 WARN [Thread-466 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:30:11,541 INFO [Thread-466 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:30:11,548 INFO [Thread-466 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_46419_jobhistory____928in8/webapp 2023-11-12T09:30:11,714 INFO [Thread-466 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-11-12T09:30:11,714 INFO [Thread-466 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-11-12T09:30:11,714 INFO [Thread-466 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-12T09:30:11,718 INFO [Thread-466 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-12T09:30:11,805 INFO [Thread-466 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:30:11,931 INFO [Thread-466 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:30:12,202 INFO [Thread-466 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-11-12T09:30:12,240 INFO [Thread-466 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:46419 2023-11-12T09:30:12,664 WARN [Listener at jenkins-hbase3.apache.org/36285 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:30:12,745 WARN [Listener at jenkins-hbase3.apache.org/36285 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-12T09:30:12,746 INFO [Listener at jenkins-hbase3.apache.org/36285 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:30:12,752 INFO [Listener at jenkins-hbase3.apache.org/36285 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_42939_cluster____5ol5cy/webapp 2023-11-12T09:30:12,824 INFO [Listener at jenkins-hbase3.apache.org/36285 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-11-12T09:30:12,824 INFO [Listener at jenkins-hbase3.apache.org/36285 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-11-12T09:30:12,825 INFO [Listener at jenkins-hbase3.apache.org/36285 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-12T09:30:12,825 INFO [Listener at jenkins-hbase3.apache.org/36285 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-12T09:30:12,833 INFO [Listener at jenkins-hbase3.apache.org/36285 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:30:13,023 INFO [Listener at jenkins-hbase3.apache.org/36285 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:30:13,106 INFO [Listener at jenkins-hbase3.apache.org/36285 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:30:13,113 INFO [Listener at jenkins-hbase3.apache.org/36285 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:42939 2023-11-12T09:30:13,555 WARN [Listener at jenkins-hbase3.apache.org/39081 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:30:13,586 WARN [Listener at jenkins-hbase3.apache.org/39081 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-12T09:30:13,586 INFO [Listener at jenkins-hbase3.apache.org/39081 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:30:13,597 INFO [Listener at jenkins-hbase3.apache.org/39081 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_44257_node____.gyqost/webapp 2023-11-12T09:30:13,649 INFO [Listener at jenkins-hbase3.apache.org/39081 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-11-12T09:30:13,649 INFO [Listener at jenkins-hbase3.apache.org/39081 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-12T09:30:13,649 INFO [Listener at jenkins-hbase3.apache.org/39081 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-11-12T09:30:13,649 INFO [Listener at jenkins-hbase3.apache.org/39081 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-12T09:30:13,659 INFO [Listener at jenkins-hbase3.apache.org/39081 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:30:13,684 INFO [Listener at jenkins-hbase3.apache.org/39081 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:30:13,786 INFO [Listener at jenkins-hbase3.apache.org/39081 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:30:13,801 INFO [Listener at jenkins-hbase3.apache.org/39081 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:44257 2023-11-12T09:30:14,067 WARN [Listener at jenkins-hbase3.apache.org/40129 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:30:14,074 WARN [Listener at jenkins-hbase3.apache.org/40129 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-12T09:30:14,075 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:30:14,083 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_46243_node____.vxtd2q/webapp 2023-11-12T09:30:14,141 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-11-12T09:30:14,141 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-12T09:30:14,141 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-11-12T09:30:14,141 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-12T09:30:14,155 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:30:14,175 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:30:14,250 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:30:14,258 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:46243 2023-11-12T09:30:14,304 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-11-12T09:30:14,304 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] backup.TestBackupBase(301): ROOTDIR hdfs://localhost:38625/backupUT 2023-11-12T09:30:14,320 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] backup.TestBackupBase(306): REMOTE ROOTDIR hdfs://localhost:41663/backupUT 2023-11-12T09:30:14,337 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:30:14,342 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:38070, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:30:14,352 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.HMaster$18(3398): Client=jenkins//172.31.12.81 creating {NAME => 'ns1'} 2023-11-12T09:30:14,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] procedure2.ProcedureExecutor(1032): Stored pid=9, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns1 2023-11-12T09:30:14,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=9 2023-11-12T09:30:14,388 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:30:14,460 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=9, state=SUCCESS; CreateNamespaceProcedure, namespace=ns1 in 105 msec 2023-11-12T09:30:14,467 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:30:14,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=9 2023-11-12T09:30:14,484 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.HMaster$18(3398): Client=jenkins//172.31.12.81 creating {NAME => 'ns2'} 2023-11-12T09:30:14,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] procedure2.ProcedureExecutor(1032): Stored pid=10, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns2 2023-11-12T09:30:14,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=10 2023-11-12T09:30:14,510 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:30:14,520 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=10, state=SUCCESS; CreateNamespaceProcedure, namespace=ns2 in 30 msec 2023-11-12T09:30:14,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=10 2023-11-12T09:30:14,612 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.HMaster$18(3398): Client=jenkins//172.31.12.81 creating {NAME => 'ns3'} 2023-11-12T09:30:14,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] procedure2.ProcedureExecutor(1032): Stored pid=11, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns3 2023-11-12T09:30:14,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=11 2023-11-12T09:30:14,635 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:30:14,645 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=11, state=SUCCESS; CreateNamespaceProcedure, namespace=ns3 in 31 msec 2023-11-12T09:30:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=11 2023-11-12T09:30:14,733 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.HMaster$18(3398): Client=jenkins//172.31.12.81 creating {NAME => 'ns4'} 2023-11-12T09:30:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] procedure2.ProcedureExecutor(1032): Stored pid=12, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns4 2023-11-12T09:30:14,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=12 2023-11-12T09:30:14,747 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:30:14,757 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=12, state=SUCCESS; CreateNamespaceProcedure, namespace=ns4 in 22 msec 2023-11-12T09:30:14,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=12 2023-11-12T09:30:14,861 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'test-1699781414320', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:30:14,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] procedure2.ProcedureExecutor(1032): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=test-1699781414320 2023-11-12T09:30:14,867 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=test-1699781414320 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:30:14,867 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "test-1699781414320" procId is: 13 2023-11-12T09:30:14,867 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:14,868 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=test-1699781414320 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:30:14,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-12T09:30:14,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-12T09:30:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-12T09:30:15,298 INFO [RegionOpenAndInit-test-1699781414320-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 37c7f0bb858c8098fd397daa8ed9cc98, NAME => 'test-1699781414320,,1699781414853.37c7f0bb858c8098fd397daa8ed9cc98.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1699781414320', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a 2023-11-12T09:30:15,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-12T09:30:15,720 DEBUG [RegionOpenAndInit-test-1699781414320-pool-0 {}] regionserver.HRegion(888): Instantiated test-1699781414320,,1699781414853.37c7f0bb858c8098fd397daa8ed9cc98.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:30:15,721 DEBUG [RegionOpenAndInit-test-1699781414320-pool-0 {}] regionserver.HRegion(1668): Closing 37c7f0bb858c8098fd397daa8ed9cc98, disabling compactions & flushes 2023-11-12T09:30:15,721 INFO [RegionOpenAndInit-test-1699781414320-pool-0 {}] regionserver.HRegion(1690): Closing region test-1699781414320,,1699781414853.37c7f0bb858c8098fd397daa8ed9cc98. 2023-11-12T09:30:15,721 DEBUG [RegionOpenAndInit-test-1699781414320-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1699781414320,,1699781414853.37c7f0bb858c8098fd397daa8ed9cc98. 2023-11-12T09:30:15,721 DEBUG [RegionOpenAndInit-test-1699781414320-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1699781414320,,1699781414853.37c7f0bb858c8098fd397daa8ed9cc98. after waiting 0 ms 2023-11-12T09:30:15,721 DEBUG [RegionOpenAndInit-test-1699781414320-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1699781414320,,1699781414853.37c7f0bb858c8098fd397daa8ed9cc98. 2023-11-12T09:30:15,721 INFO [RegionOpenAndInit-test-1699781414320-pool-0 {}] regionserver.HRegion(1909): Closed test-1699781414320,,1699781414853.37c7f0bb858c8098fd397daa8ed9cc98. 2023-11-12T09:30:15,721 DEBUG [RegionOpenAndInit-test-1699781414320-pool-0 {}] regionserver.HRegion(1622): Region close journal for 37c7f0bb858c8098fd397daa8ed9cc98: 2023-11-12T09:30:15,723 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=test-1699781414320 execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:30:15,723 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1699781414320,,1699781414853.37c7f0bb858c8098fd397daa8ed9cc98.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1699781415723"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781415723"}]},"ts":"1699781415723"} 2023-11-12T09:30:15,726 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:30:15,727 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=test-1699781414320 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:30:15,727 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1699781414320","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781415727"}]},"ts":"1699781415727"} 2023-11-12T09:30:15,729 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1699781414320, state=ENABLING in hbase:meta 2023-11-12T09:30:15,733 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1699781414320, region=37c7f0bb858c8098fd397daa8ed9cc98, ASSIGN}] 2023-11-12T09:30:15,736 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1699781414320, region=37c7f0bb858c8098fd397daa8ed9cc98, ASSIGN 2023-11-12T09:30:15,737 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1699781414320, region=37c7f0bb858c8098fd397daa8ed9cc98, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37573,1699781400943; forceNewPlan=false, retain=false 2023-11-12T09:30:15,889 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=37c7f0bb858c8098fd397daa8ed9cc98, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:15,892 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; OpenRegionProcedure 37c7f0bb858c8098fd397daa8ed9cc98, server=jenkins-hbase3.apache.org,37573,1699781400943}] 2023-11-12T09:30:15,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-12T09:30:16,050 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:16,062 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(130): Open test-1699781414320,,1699781414853.37c7f0bb858c8098fd397daa8ed9cc98. 2023-11-12T09:30:16,062 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7254): Opening region: {ENCODED => 37c7f0bb858c8098fd397daa8ed9cc98, NAME => 'test-1699781414320,,1699781414853.37c7f0bb858c8098fd397daa8ed9cc98.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:30:16,062 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:30:16,063 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1699781414320 37c7f0bb858c8098fd397daa8ed9cc98 2023-11-12T09:30:16,063 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(888): Instantiated test-1699781414320,,1699781414853.37c7f0bb858c8098fd397daa8ed9cc98.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:30:16,063 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7296): checking encryption for 37c7f0bb858c8098fd397daa8ed9cc98 2023-11-12T09:30:16,063 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7299): checking classloading for 37c7f0bb858c8098fd397daa8ed9cc98 2023-11-12T09:30:16,065 INFO [StoreOpener-37c7f0bb858c8098fd397daa8ed9cc98-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 37c7f0bb858c8098fd397daa8ed9cc98 2023-11-12T09:30:16,067 INFO [StoreOpener-37c7f0bb858c8098fd397daa8ed9cc98-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 37c7f0bb858c8098fd397daa8ed9cc98 columnFamilyName f 2023-11-12T09:30:16,067 DEBUG [StoreOpener-37c7f0bb858c8098fd397daa8ed9cc98-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:16,068 INFO [StoreOpener-37c7f0bb858c8098fd397daa8ed9cc98-1 {}] regionserver.HStore(324): Store=37c7f0bb858c8098fd397daa8ed9cc98/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:30:16,069 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/test-1699781414320/37c7f0bb858c8098fd397daa8ed9cc98 2023-11-12T09:30:16,070 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/test-1699781414320/37c7f0bb858c8098fd397daa8ed9cc98 2023-11-12T09:30:16,073 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1076): writing seq id for 37c7f0bb858c8098fd397daa8ed9cc98 2023-11-12T09:30:16,076 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/test-1699781414320/37c7f0bb858c8098fd397daa8ed9cc98/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:30:16,077 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): Opened 37c7f0bb858c8098fd397daa8ed9cc98; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10121724640, jitterRate=-0.05734093487262726}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:30:16,077 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(992): Region open journal for 37c7f0bb858c8098fd397daa8ed9cc98: 2023-11-12T09:30:16,079 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2556): Post open deploy tasks for test-1699781414320,,1699781414853.37c7f0bb858c8098fd397daa8ed9cc98., pid=15, masterSystemTime=1699781416050 2023-11-12T09:30:16,081 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2583): Finished post open deploy task for test-1699781414320,,1699781414853.37c7f0bb858c8098fd397daa8ed9cc98. 2023-11-12T09:30:16,081 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(158): Opened test-1699781414320,,1699781414853.37c7f0bb858c8098fd397daa8ed9cc98. 2023-11-12T09:30:16,082 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=37c7f0bb858c8098fd397daa8ed9cc98, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:16,087 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=15, resume processing ppid=14 2023-11-12T09:30:16,088 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=15, ppid=14, state=SUCCESS; OpenRegionProcedure 37c7f0bb858c8098fd397daa8ed9cc98, server=jenkins-hbase3.apache.org,37573,1699781400943 in 192 msec 2023-11-12T09:30:16,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=14, resume processing ppid=13 2023-11-12T09:30:16,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=14, ppid=13, state=SUCCESS; TransitRegionStateProcedure table=test-1699781414320, region=37c7f0bb858c8098fd397daa8ed9cc98, ASSIGN in 354 msec 2023-11-12T09:30:16,091 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=test-1699781414320 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:30:16,091 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1699781414320","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781416091"}]},"ts":"1699781416091"} 2023-11-12T09:30:16,093 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1699781414320, state=ENABLED in hbase:meta 2023-11-12T09:30:16,096 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=test-1699781414320 execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:30:16,098 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=13, state=SUCCESS; CreateTableProcedure table=test-1699781414320 in 1.2350 sec 2023-11-12T09:30:16,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-12T09:30:16,984 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: default:test-1699781414320, procId: 13 completed 2023-11-12T09:30:16,986 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x79aff875 to 127.0.0.1:57951 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:30:16,994 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b63d83b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:30:16,996 DEBUG [hconnection-0x1baee58e-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:30:17,000 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:43434, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:30:17,019 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] regionserver.HRegion(8210): writing data to region test-1699781414320,,1699781414853.37c7f0bb858c8098fd397daa8ed9cc98. with WAL disabled. Data may be lost in the event of a crash. 2023-11-12T09:30:17,120 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'ns2:test-16997814143201', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:30:17,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] procedure2.ProcedureExecutor(1032): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:test-16997814143201 2023-11-12T09:30:17,125 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16997814143201 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:30:17,125 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:17,125 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "test-16997814143201" procId is: 16 2023-11-12T09:30:17,126 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:test-16997814143201 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:30:17,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-12T09:30:17,141 INFO [RegionOpenAndInit-ns2:test-16997814143201-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 7dc83426edfa8b37518a7f14a9649f4c, NAME => 'ns2:test-16997814143201,,1699781417119.7dc83426edfa8b37518a7f14a9649f4c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16997814143201', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a 2023-11-12T09:30:17,157 DEBUG [RegionOpenAndInit-ns2:test-16997814143201-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16997814143201,,1699781417119.7dc83426edfa8b37518a7f14a9649f4c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:30:17,157 DEBUG [RegionOpenAndInit-ns2:test-16997814143201-pool-0 {}] regionserver.HRegion(1668): Closing 7dc83426edfa8b37518a7f14a9649f4c, disabling compactions & flushes 2023-11-12T09:30:17,157 INFO [RegionOpenAndInit-ns2:test-16997814143201-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16997814143201,,1699781417119.7dc83426edfa8b37518a7f14a9649f4c. 2023-11-12T09:30:17,157 DEBUG [RegionOpenAndInit-ns2:test-16997814143201-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16997814143201,,1699781417119.7dc83426edfa8b37518a7f14a9649f4c. 2023-11-12T09:30:17,157 DEBUG [RegionOpenAndInit-ns2:test-16997814143201-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16997814143201,,1699781417119.7dc83426edfa8b37518a7f14a9649f4c. after waiting 0 ms 2023-11-12T09:30:17,157 DEBUG [RegionOpenAndInit-ns2:test-16997814143201-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16997814143201,,1699781417119.7dc83426edfa8b37518a7f14a9649f4c. 2023-11-12T09:30:17,157 INFO [RegionOpenAndInit-ns2:test-16997814143201-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16997814143201,,1699781417119.7dc83426edfa8b37518a7f14a9649f4c. 2023-11-12T09:30:17,157 DEBUG [RegionOpenAndInit-ns2:test-16997814143201-pool-0 {}] regionserver.HRegion(1622): Region close journal for 7dc83426edfa8b37518a7f14a9649f4c: 2023-11-12T09:30:17,159 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:test-16997814143201 execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:30:17,159 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16997814143201,,1699781417119.7dc83426edfa8b37518a7f14a9649f4c.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1699781417159"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781417159"}]},"ts":"1699781417159"} 2023-11-12T09:30:17,162 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:30:17,163 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:test-16997814143201 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:30:17,163 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16997814143201","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781417163"}]},"ts":"1699781417163"} 2023-11-12T09:30:17,165 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16997814143201, state=ENABLING in hbase:meta 2023-11-12T09:30:17,169 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16997814143201, region=7dc83426edfa8b37518a7f14a9649f4c, ASSIGN}] 2023-11-12T09:30:17,179 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16997814143201, region=7dc83426edfa8b37518a7f14a9649f4c, ASSIGN 2023-11-12T09:30:17,181 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16997814143201, region=7dc83426edfa8b37518a7f14a9649f4c, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37573,1699781400943; forceNewPlan=false, retain=false 2023-11-12T09:30:17,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-12T09:30:17,333 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=7dc83426edfa8b37518a7f14a9649f4c, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:17,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; OpenRegionProcedure 7dc83426edfa8b37518a7f14a9649f4c, server=jenkins-hbase3.apache.org,37573,1699781400943}] 2023-11-12T09:30:17,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-12T09:30:17,488 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:17,493 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(130): Open ns2:test-16997814143201,,1699781417119.7dc83426edfa8b37518a7f14a9649f4c. 2023-11-12T09:30:17,493 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7254): Opening region: {ENCODED => 7dc83426edfa8b37518a7f14a9649f4c, NAME => 'ns2:test-16997814143201,,1699781417119.7dc83426edfa8b37518a7f14a9649f4c.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:30:17,494 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:30:17,494 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16997814143201 7dc83426edfa8b37518a7f14a9649f4c 2023-11-12T09:30:17,494 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(888): Instantiated ns2:test-16997814143201,,1699781417119.7dc83426edfa8b37518a7f14a9649f4c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:30:17,494 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7296): checking encryption for 7dc83426edfa8b37518a7f14a9649f4c 2023-11-12T09:30:17,494 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7299): checking classloading for 7dc83426edfa8b37518a7f14a9649f4c 2023-11-12T09:30:17,496 INFO [StoreOpener-7dc83426edfa8b37518a7f14a9649f4c-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 7dc83426edfa8b37518a7f14a9649f4c 2023-11-12T09:30:17,497 INFO [StoreOpener-7dc83426edfa8b37518a7f14a9649f4c-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7dc83426edfa8b37518a7f14a9649f4c columnFamilyName f 2023-11-12T09:30:17,497 DEBUG [StoreOpener-7dc83426edfa8b37518a7f14a9649f4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:17,498 INFO [StoreOpener-7dc83426edfa8b37518a7f14a9649f4c-1 {}] regionserver.HStore(324): Store=7dc83426edfa8b37518a7f14a9649f4c/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:30:17,499 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/ns2/test-16997814143201/7dc83426edfa8b37518a7f14a9649f4c 2023-11-12T09:30:17,499 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/ns2/test-16997814143201/7dc83426edfa8b37518a7f14a9649f4c 2023-11-12T09:30:17,503 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1076): writing seq id for 7dc83426edfa8b37518a7f14a9649f4c 2023-11-12T09:30:17,506 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/ns2/test-16997814143201/7dc83426edfa8b37518a7f14a9649f4c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:30:17,506 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): Opened 7dc83426edfa8b37518a7f14a9649f4c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9518989600, jitterRate=-0.1134750097990036}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:30:17,507 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(992): Region open journal for 7dc83426edfa8b37518a7f14a9649f4c: 2023-11-12T09:30:17,509 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:test-16997814143201,,1699781417119.7dc83426edfa8b37518a7f14a9649f4c., pid=18, masterSystemTime=1699781417488 2023-11-12T09:30:17,511 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:test-16997814143201,,1699781417119.7dc83426edfa8b37518a7f14a9649f4c. 2023-11-12T09:30:17,511 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(158): Opened ns2:test-16997814143201,,1699781417119.7dc83426edfa8b37518a7f14a9649f4c. 2023-11-12T09:30:17,512 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=7dc83426edfa8b37518a7f14a9649f4c, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:17,524 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=18, resume processing ppid=17 2023-11-12T09:30:17,524 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=18, ppid=17, state=SUCCESS; OpenRegionProcedure 7dc83426edfa8b37518a7f14a9649f4c, server=jenkins-hbase3.apache.org,37573,1699781400943 in 187 msec 2023-11-12T09:30:17,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=17, resume processing ppid=16 2023-11-12T09:30:17,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=17, ppid=16, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16997814143201, region=7dc83426edfa8b37518a7f14a9649f4c, ASSIGN in 355 msec 2023-11-12T09:30:17,528 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:test-16997814143201 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:30:17,528 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16997814143201","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781417528"}]},"ts":"1699781417528"} 2023-11-12T09:30:17,531 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16997814143201, state=ENABLED in hbase:meta 2023-11-12T09:30:17,534 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16997814143201 execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:30:17,535 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=16, state=SUCCESS; CreateTableProcedure table=ns2:test-16997814143201 in 414 msec 2023-11-12T09:30:17,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-12T09:30:17,731 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: ns2:test-16997814143201, procId: 16 completed 2023-11-12T09:30:17,737 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] regionserver.HRegion(8210): writing data to region ns2:test-16997814143201,,1699781417119.7dc83426edfa8b37518a7f14a9649f4c. with WAL disabled. Data may be lost in the event of a crash. 2023-11-12T09:30:17,817 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'ns3:test-16997814143202', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:30:17,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] procedure2.ProcedureExecutor(1032): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:test-16997814143202 2023-11-12T09:30:17,821 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16997814143202 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:30:17,821 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:17,821 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "test-16997814143202" procId is: 19 2023-11-12T09:30:17,822 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:test-16997814143202 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:30:17,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-12T09:30:17,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-12T09:30:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-12T09:30:18,245 INFO [RegionOpenAndInit-ns3:test-16997814143202-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => e2e74d85dcb767a4afcaf7637dfbf1b3, NAME => 'ns3:test-16997814143202,,1699781417816.e2e74d85dcb767a4afcaf7637dfbf1b3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16997814143202', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a 2023-11-12T09:30:18,263 DEBUG [RegionOpenAndInit-ns3:test-16997814143202-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16997814143202,,1699781417816.e2e74d85dcb767a4afcaf7637dfbf1b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:30:18,263 DEBUG [RegionOpenAndInit-ns3:test-16997814143202-pool-0 {}] regionserver.HRegion(1668): Closing e2e74d85dcb767a4afcaf7637dfbf1b3, disabling compactions & flushes 2023-11-12T09:30:18,263 INFO [RegionOpenAndInit-ns3:test-16997814143202-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16997814143202,,1699781417816.e2e74d85dcb767a4afcaf7637dfbf1b3. 2023-11-12T09:30:18,263 DEBUG [RegionOpenAndInit-ns3:test-16997814143202-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16997814143202,,1699781417816.e2e74d85dcb767a4afcaf7637dfbf1b3. 2023-11-12T09:30:18,263 DEBUG [RegionOpenAndInit-ns3:test-16997814143202-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16997814143202,,1699781417816.e2e74d85dcb767a4afcaf7637dfbf1b3. after waiting 0 ms 2023-11-12T09:30:18,263 DEBUG [RegionOpenAndInit-ns3:test-16997814143202-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16997814143202,,1699781417816.e2e74d85dcb767a4afcaf7637dfbf1b3. 2023-11-12T09:30:18,263 INFO [RegionOpenAndInit-ns3:test-16997814143202-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16997814143202,,1699781417816.e2e74d85dcb767a4afcaf7637dfbf1b3. 2023-11-12T09:30:18,263 DEBUG [RegionOpenAndInit-ns3:test-16997814143202-pool-0 {}] regionserver.HRegion(1622): Region close journal for e2e74d85dcb767a4afcaf7637dfbf1b3: 2023-11-12T09:30:18,265 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:test-16997814143202 execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:30:18,265 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16997814143202,,1699781417816.e2e74d85dcb767a4afcaf7637dfbf1b3.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1699781418265"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781418265"}]},"ts":"1699781418265"} 2023-11-12T09:30:18,269 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:30:18,270 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:test-16997814143202 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:30:18,270 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16997814143202","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781418270"}]},"ts":"1699781418270"} 2023-11-12T09:30:18,272 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16997814143202, state=ENABLING in hbase:meta 2023-11-12T09:30:18,277 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16997814143202, region=e2e74d85dcb767a4afcaf7637dfbf1b3, ASSIGN}] 2023-11-12T09:30:18,280 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16997814143202, region=e2e74d85dcb767a4afcaf7637dfbf1b3, ASSIGN 2023-11-12T09:30:18,281 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16997814143202, region=e2e74d85dcb767a4afcaf7637dfbf1b3, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37573,1699781400943; forceNewPlan=false, retain=false 2023-11-12T09:30:18,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-12T09:30:18,433 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=e2e74d85dcb767a4afcaf7637dfbf1b3, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:18,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; OpenRegionProcedure e2e74d85dcb767a4afcaf7637dfbf1b3, server=jenkins-hbase3.apache.org,37573,1699781400943}] 2023-11-12T09:30:18,588 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:18,593 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(130): Open ns3:test-16997814143202,,1699781417816.e2e74d85dcb767a4afcaf7637dfbf1b3. 2023-11-12T09:30:18,594 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7254): Opening region: {ENCODED => e2e74d85dcb767a4afcaf7637dfbf1b3, NAME => 'ns3:test-16997814143202,,1699781417816.e2e74d85dcb767a4afcaf7637dfbf1b3.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:30:18,594 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:30:18,594 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16997814143202 e2e74d85dcb767a4afcaf7637dfbf1b3 2023-11-12T09:30:18,594 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(888): Instantiated ns3:test-16997814143202,,1699781417816.e2e74d85dcb767a4afcaf7637dfbf1b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:30:18,594 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7296): checking encryption for e2e74d85dcb767a4afcaf7637dfbf1b3 2023-11-12T09:30:18,594 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7299): checking classloading for e2e74d85dcb767a4afcaf7637dfbf1b3 2023-11-12T09:30:18,596 INFO [StoreOpener-e2e74d85dcb767a4afcaf7637dfbf1b3-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region e2e74d85dcb767a4afcaf7637dfbf1b3 2023-11-12T09:30:18,598 INFO [StoreOpener-e2e74d85dcb767a4afcaf7637dfbf1b3-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2e74d85dcb767a4afcaf7637dfbf1b3 columnFamilyName f 2023-11-12T09:30:18,598 DEBUG [StoreOpener-e2e74d85dcb767a4afcaf7637dfbf1b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:18,598 INFO [StoreOpener-e2e74d85dcb767a4afcaf7637dfbf1b3-1 {}] regionserver.HStore(324): Store=e2e74d85dcb767a4afcaf7637dfbf1b3/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:30:18,600 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/ns3/test-16997814143202/e2e74d85dcb767a4afcaf7637dfbf1b3 2023-11-12T09:30:18,600 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/ns3/test-16997814143202/e2e74d85dcb767a4afcaf7637dfbf1b3 2023-11-12T09:30:18,604 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1076): writing seq id for e2e74d85dcb767a4afcaf7637dfbf1b3 2023-11-12T09:30:18,608 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/ns3/test-16997814143202/e2e74d85dcb767a4afcaf7637dfbf1b3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:30:18,609 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): Opened e2e74d85dcb767a4afcaf7637dfbf1b3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11331188800, jitterRate=0.055299192667007446}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:30:18,610 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(992): Region open journal for e2e74d85dcb767a4afcaf7637dfbf1b3: 2023-11-12T09:30:18,611 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:test-16997814143202,,1699781417816.e2e74d85dcb767a4afcaf7637dfbf1b3., pid=21, masterSystemTime=1699781418588 2023-11-12T09:30:18,613 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:test-16997814143202,,1699781417816.e2e74d85dcb767a4afcaf7637dfbf1b3. 2023-11-12T09:30:18,613 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(158): Opened ns3:test-16997814143202,,1699781417816.e2e74d85dcb767a4afcaf7637dfbf1b3. 2023-11-12T09:30:18,614 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=e2e74d85dcb767a4afcaf7637dfbf1b3, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:18,619 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=21, resume processing ppid=20 2023-11-12T09:30:18,619 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=21, ppid=20, state=SUCCESS; OpenRegionProcedure e2e74d85dcb767a4afcaf7637dfbf1b3, server=jenkins-hbase3.apache.org,37573,1699781400943 in 180 msec 2023-11-12T09:30:18,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=20, resume processing ppid=19 2023-11-12T09:30:18,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=20, ppid=19, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16997814143202, region=e2e74d85dcb767a4afcaf7637dfbf1b3, ASSIGN in 342 msec 2023-11-12T09:30:18,626 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:test-16997814143202 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:30:18,626 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16997814143202","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781418626"}]},"ts":"1699781418626"} 2023-11-12T09:30:18,628 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16997814143202, state=ENABLED in hbase:meta 2023-11-12T09:30:18,631 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16997814143202 execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:30:18,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=19, state=SUCCESS; CreateTableProcedure table=ns3:test-16997814143202 in 814 msec 2023-11-12T09:30:18,915 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1699781414320 2023-11-12T09:30:18,915 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1699781414320 Metrics about Tables on a single HBase RegionServer 2023-11-12T09:30:18,916 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16997814143202 2023-11-12T09:30:18,916 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-16997814143202 Metrics about Tables on a single HBase RegionServer 2023-11-12T09:30:18,917 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16997814143201 2023-11-12T09:30:18,917 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-16997814143201 Metrics about Tables on a single HBase RegionServer 2023-11-12T09:30:18,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-12T09:30:18,933 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: ns3:test-16997814143202, procId: 19 completed 2023-11-12T09:30:18,933 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns3:test-16997814143202 get assigned. Timeout = 60000ms 2023-11-12T09:30:18,935 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-12T09:30:18,939 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.HBaseTestingUtility(3581): All regions for table ns3:test-16997814143202 assigned to meta. Checking AM states. 2023-11-12T09:30:18,940 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-12T09:30:18,940 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.HBaseTestingUtility(3601): All regions for table ns3:test-16997814143202 assigned. 2023-11-12T09:30:18,942 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'ns4:test-16997814143203', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:30:18,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] procedure2.ProcedureExecutor(1032): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns4:test-16997814143203 2023-11-12T09:30:18,950 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16997814143203 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:30:18,950 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:18,950 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns4" qualifier: "test-16997814143203" procId is: 22 2023-11-12T09:30:18,952 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns4:test-16997814143203 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:30:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-12T09:30:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-12T09:30:19,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-12T09:30:19,375 INFO [RegionOpenAndInit-ns4:test-16997814143203-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => e4f203e272def65afc0882551ad2e3de, NAME => 'ns4:test-16997814143203,,1699781418942.e4f203e272def65afc0882551ad2e3de.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-16997814143203', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a 2023-11-12T09:30:19,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-12T09:30:19,790 DEBUG [RegionOpenAndInit-ns4:test-16997814143203-pool-0 {}] regionserver.HRegion(888): Instantiated ns4:test-16997814143203,,1699781418942.e4f203e272def65afc0882551ad2e3de.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:30:19,791 DEBUG [RegionOpenAndInit-ns4:test-16997814143203-pool-0 {}] regionserver.HRegion(1668): Closing e4f203e272def65afc0882551ad2e3de, disabling compactions & flushes 2023-11-12T09:30:19,791 INFO [RegionOpenAndInit-ns4:test-16997814143203-pool-0 {}] regionserver.HRegion(1690): Closing region ns4:test-16997814143203,,1699781418942.e4f203e272def65afc0882551ad2e3de. 2023-11-12T09:30:19,791 DEBUG [RegionOpenAndInit-ns4:test-16997814143203-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16997814143203,,1699781418942.e4f203e272def65afc0882551ad2e3de. 2023-11-12T09:30:19,791 DEBUG [RegionOpenAndInit-ns4:test-16997814143203-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16997814143203,,1699781418942.e4f203e272def65afc0882551ad2e3de. after waiting 0 ms 2023-11-12T09:30:19,791 DEBUG [RegionOpenAndInit-ns4:test-16997814143203-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16997814143203,,1699781418942.e4f203e272def65afc0882551ad2e3de. 2023-11-12T09:30:19,791 INFO [RegionOpenAndInit-ns4:test-16997814143203-pool-0 {}] regionserver.HRegion(1909): Closed ns4:test-16997814143203,,1699781418942.e4f203e272def65afc0882551ad2e3de. 2023-11-12T09:30:19,791 DEBUG [RegionOpenAndInit-ns4:test-16997814143203-pool-0 {}] regionserver.HRegion(1622): Region close journal for e4f203e272def65afc0882551ad2e3de: 2023-11-12T09:30:19,793 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns4:test-16997814143203 execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:30:19,793 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns4:test-16997814143203,,1699781418942.e4f203e272def65afc0882551ad2e3de.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1699781419793"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781419793"}]},"ts":"1699781419793"} 2023-11-12T09:30:19,796 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:30:19,797 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns4:test-16997814143203 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:30:19,797 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16997814143203","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781419797"}]},"ts":"1699781419797"} 2023-11-12T09:30:19,799 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16997814143203, state=ENABLING in hbase:meta 2023-11-12T09:30:19,804 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16997814143203, region=e4f203e272def65afc0882551ad2e3de, ASSIGN}] 2023-11-12T09:30:19,808 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16997814143203, region=e4f203e272def65afc0882551ad2e3de, ASSIGN 2023-11-12T09:30:19,809 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns4:test-16997814143203, region=e4f203e272def65afc0882551ad2e3de, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37573,1699781400943; forceNewPlan=false, retain=false 2023-11-12T09:30:19,961 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=e4f203e272def65afc0882551ad2e3de, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:19,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE; OpenRegionProcedure e4f203e272def65afc0882551ad2e3de, server=jenkins-hbase3.apache.org,37573,1699781400943}] 2023-11-12T09:30:20,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-12T09:30:20,116 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:20,121 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(130): Open ns4:test-16997814143203,,1699781418942.e4f203e272def65afc0882551ad2e3de. 2023-11-12T09:30:20,121 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7254): Opening region: {ENCODED => e4f203e272def65afc0882551ad2e3de, NAME => 'ns4:test-16997814143203,,1699781418942.e4f203e272def65afc0882551ad2e3de.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:30:20,121 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:30:20,121 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16997814143203 e4f203e272def65afc0882551ad2e3de 2023-11-12T09:30:20,121 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(888): Instantiated ns4:test-16997814143203,,1699781418942.e4f203e272def65afc0882551ad2e3de.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:30:20,121 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7296): checking encryption for e4f203e272def65afc0882551ad2e3de 2023-11-12T09:30:20,122 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7299): checking classloading for e4f203e272def65afc0882551ad2e3de 2023-11-12T09:30:20,123 INFO [StoreOpener-e4f203e272def65afc0882551ad2e3de-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region e4f203e272def65afc0882551ad2e3de 2023-11-12T09:30:20,125 INFO [StoreOpener-e4f203e272def65afc0882551ad2e3de-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4f203e272def65afc0882551ad2e3de columnFamilyName f 2023-11-12T09:30:20,125 DEBUG [StoreOpener-e4f203e272def65afc0882551ad2e3de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:20,125 INFO [StoreOpener-e4f203e272def65afc0882551ad2e3de-1 {}] regionserver.HStore(324): Store=e4f203e272def65afc0882551ad2e3de/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:30:20,126 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/ns4/test-16997814143203/e4f203e272def65afc0882551ad2e3de 2023-11-12T09:30:20,127 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/ns4/test-16997814143203/e4f203e272def65afc0882551ad2e3de 2023-11-12T09:30:20,133 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1076): writing seq id for e4f203e272def65afc0882551ad2e3de 2023-11-12T09:30:20,139 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/ns4/test-16997814143203/e4f203e272def65afc0882551ad2e3de/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:30:20,139 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): Opened e4f203e272def65afc0882551ad2e3de; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9532503360, jitterRate=-0.11221644282341003}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:30:20,140 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(992): Region open journal for e4f203e272def65afc0882551ad2e3de: 2023-11-12T09:30:20,141 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2556): Post open deploy tasks for ns4:test-16997814143203,,1699781418942.e4f203e272def65afc0882551ad2e3de., pid=24, masterSystemTime=1699781420116 2023-11-12T09:30:20,143 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2583): Finished post open deploy task for ns4:test-16997814143203,,1699781418942.e4f203e272def65afc0882551ad2e3de. 2023-11-12T09:30:20,143 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(158): Opened ns4:test-16997814143203,,1699781418942.e4f203e272def65afc0882551ad2e3de. 2023-11-12T09:30:20,144 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=e4f203e272def65afc0882551ad2e3de, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:20,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=24, resume processing ppid=23 2023-11-12T09:30:20,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=24, ppid=23, state=SUCCESS; OpenRegionProcedure e4f203e272def65afc0882551ad2e3de, server=jenkins-hbase3.apache.org,37573,1699781400943 in 183 msec 2023-11-12T09:30:20,150 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=23, resume processing ppid=22 2023-11-12T09:30:20,150 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=23, ppid=22, state=SUCCESS; TransitRegionStateProcedure table=ns4:test-16997814143203, region=e4f203e272def65afc0882551ad2e3de, ASSIGN in 344 msec 2023-11-12T09:30:20,151 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns4:test-16997814143203 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:30:20,151 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16997814143203","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781420151"}]},"ts":"1699781420151"} 2023-11-12T09:30:20,153 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16997814143203, state=ENABLED in hbase:meta 2023-11-12T09:30:20,155 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16997814143203 execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:30:20,157 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=22, state=SUCCESS; CreateTableProcedure table=ns4:test-16997814143203 in 1.2120 sec 2023-11-12T09:30:21,006 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-16997814143201' 2023-11-12T09:30:21,007 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-16997814143203' 2023-11-12T09:30:21,007 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1699781414320' 2023-11-12T09:30:21,007 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-16997814143202' 2023-11-12T09:30:21,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-12T09:30:21,062 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: ns4:test-16997814143203, procId: 22 completed 2023-11-12T09:30:21,062 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns4:test-16997814143203 get assigned. Timeout = 60000ms 2023-11-12T09:30:21,062 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-12T09:30:21,068 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.HBaseTestingUtility(3581): All regions for table ns4:test-16997814143203 assigned to meta. Checking AM states. 2023-11-12T09:30:21,068 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-12T09:30:21,068 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.HBaseTestingUtility(3601): All regions for table ns4:test-16997814143203 assigned. 2023-11-12T09:30:21,068 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x79aff875 to 127.0.0.1:57951 2023-11-12T09:30:21,069 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:30:21,131 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.ResourceChecker(147): before: backup.TestRemoteRestore#testFullRestoreRemote Thread=814, OpenFileDescriptor=907, MaxFileDescriptor=60000, SystemLoadAverage=163, ProcessCount=169, AvailableMemoryMB=5587 2023-11-12T09:30:21,131 WARN [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.ResourceChecker(130): Thread=814 is superior to 500 2023-11-12T09:30:21,133 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] backup.TestRemoteRestore(70): test remote full backup on a single table 2023-11-12T09:30:21,135 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x708a33bc to 127.0.0.1:57951 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:30:21,141 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47d2755b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:30:21,165 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:30:21,167 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:43446, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:30:21,180 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:30:21,182 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:54360, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:30:21,192 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.HMaster$18(3398): Client=jenkins//172.31.12.81 creating {NAME => 'backup'} 2023-11-12T09:30:21,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] procedure2.ProcedureExecutor(1032): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=backup 2023-11-12T09:30:21,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=25 2023-11-12T09:30:21,204 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:30:21,207 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=25, state=SUCCESS; CreateNamespaceProcedure, namespace=backup in 14 msec 2023-11-12T09:30:21,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=25 2023-11-12T09:30:21,319 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:30:21,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] procedure2.ProcedureExecutor(1032): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system 2023-11-12T09:30:21,326 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:30:21,326 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:21,326 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2023-11-12T09:30:21,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-12T09:30:21,327 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:30:21,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-12T09:30:21,567 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:30:21,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-12T09:30:21,760 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 441fc40990b285ba0c11d5ae2272b2af, NAME => 'backup:system,,1699781421319.441fc40990b285ba0c11d5ae2272b2af.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a 2023-11-12T09:30:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-12T09:30:22,173 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system,,1699781421319.441fc40990b285ba0c11d5ae2272b2af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:30:22,173 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1668): Closing 441fc40990b285ba0c11d5ae2272b2af, disabling compactions & flushes 2023-11-12T09:30:22,173 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system,,1699781421319.441fc40990b285ba0c11d5ae2272b2af. 2023-11-12T09:30:22,173 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1699781421319.441fc40990b285ba0c11d5ae2272b2af. 2023-11-12T09:30:22,173 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1699781421319.441fc40990b285ba0c11d5ae2272b2af. after waiting 0 ms 2023-11-12T09:30:22,173 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1699781421319.441fc40990b285ba0c11d5ae2272b2af. 2023-11-12T09:30:22,173 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1909): Closed backup:system,,1699781421319.441fc40990b285ba0c11d5ae2272b2af. 2023-11-12T09:30:22,173 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1622): Region close journal for 441fc40990b285ba0c11d5ae2272b2af: 2023-11-12T09:30:22,175 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:30:22,175 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1699781421319.441fc40990b285ba0c11d5ae2272b2af.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1699781422175"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781422175"}]},"ts":"1699781422175"} 2023-11-12T09:30:22,177 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:30:22,178 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:30:22,179 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781422178"}]},"ts":"1699781422178"} 2023-11-12T09:30:22,180 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-11-12T09:30:22,184 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=441fc40990b285ba0c11d5ae2272b2af, ASSIGN}] 2023-11-12T09:30:22,186 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=441fc40990b285ba0c11d5ae2272b2af, ASSIGN 2023-11-12T09:30:22,187 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=441fc40990b285ba0c11d5ae2272b2af, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37573,1699781400943; forceNewPlan=false, retain=false 2023-11-12T09:30:22,339 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=441fc40990b285ba0c11d5ae2272b2af, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:22,341 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; OpenRegionProcedure 441fc40990b285ba0c11d5ae2272b2af, server=jenkins-hbase3.apache.org,37573,1699781400943}] 2023-11-12T09:30:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-12T09:30:22,494 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:22,498 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(130): Open backup:system,,1699781421319.441fc40990b285ba0c11d5ae2272b2af. 2023-11-12T09:30:22,498 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7254): Opening region: {ENCODED => 441fc40990b285ba0c11d5ae2272b2af, NAME => 'backup:system,,1699781421319.441fc40990b285ba0c11d5ae2272b2af.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:30:22,498 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:30:22,498 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 441fc40990b285ba0c11d5ae2272b2af 2023-11-12T09:30:22,498 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(888): Instantiated backup:system,,1699781421319.441fc40990b285ba0c11d5ae2272b2af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:30:22,498 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7296): checking encryption for 441fc40990b285ba0c11d5ae2272b2af 2023-11-12T09:30:22,498 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7299): checking classloading for 441fc40990b285ba0c11d5ae2272b2af 2023-11-12T09:30:22,500 INFO [StoreOpener-441fc40990b285ba0c11d5ae2272b2af-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 441fc40990b285ba0c11d5ae2272b2af 2023-11-12T09:30:22,502 INFO [StoreOpener-441fc40990b285ba0c11d5ae2272b2af-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 441fc40990b285ba0c11d5ae2272b2af columnFamilyName meta 2023-11-12T09:30:22,502 DEBUG [StoreOpener-441fc40990b285ba0c11d5ae2272b2af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:22,503 INFO [StoreOpener-441fc40990b285ba0c11d5ae2272b2af-1 {}] regionserver.HStore(324): Store=441fc40990b285ba0c11d5ae2272b2af/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:30:22,503 INFO [StoreOpener-441fc40990b285ba0c11d5ae2272b2af-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 441fc40990b285ba0c11d5ae2272b2af 2023-11-12T09:30:22,504 INFO [StoreOpener-441fc40990b285ba0c11d5ae2272b2af-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 441fc40990b285ba0c11d5ae2272b2af columnFamilyName session 2023-11-12T09:30:22,504 DEBUG [StoreOpener-441fc40990b285ba0c11d5ae2272b2af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:22,505 INFO [StoreOpener-441fc40990b285ba0c11d5ae2272b2af-1 {}] regionserver.HStore(324): Store=441fc40990b285ba0c11d5ae2272b2af/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:30:22,506 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af 2023-11-12T09:30:22,506 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af 2023-11-12T09:30:22,508 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-11-12T09:30:22,509 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1076): writing seq id for 441fc40990b285ba0c11d5ae2272b2af 2023-11-12T09:30:22,512 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:30:22,512 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): Opened 441fc40990b285ba0c11d5ae2272b2af; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11570912800, jitterRate=0.07762522995471954}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-11-12T09:30:22,513 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(992): Region open journal for 441fc40990b285ba0c11d5ae2272b2af: 2023-11-12T09:30:22,514 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1699781421319.441fc40990b285ba0c11d5ae2272b2af., pid=28, masterSystemTime=1699781422494 2023-11-12T09:30:22,516 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1699781421319.441fc40990b285ba0c11d5ae2272b2af. 2023-11-12T09:30:22,516 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(158): Opened backup:system,,1699781421319.441fc40990b285ba0c11d5ae2272b2af. 2023-11-12T09:30:22,516 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=441fc40990b285ba0c11d5ae2272b2af, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:22,520 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=28, resume processing ppid=27 2023-11-12T09:30:22,520 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=28, ppid=27, state=SUCCESS; OpenRegionProcedure 441fc40990b285ba0c11d5ae2272b2af, server=jenkins-hbase3.apache.org,37573,1699781400943 in 177 msec 2023-11-12T09:30:22,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=27, resume processing ppid=26 2023-11-12T09:30:22,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=27, ppid=26, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=441fc40990b285ba0c11d5ae2272b2af, ASSIGN in 336 msec 2023-11-12T09:30:22,523 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:30:22,524 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781422523"}]},"ts":"1699781422523"} 2023-11-12T09:30:22,525 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-11-12T09:30:22,529 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:30:22,531 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=26, state=SUCCESS; CreateTableProcedure table=backup:system in 1.2100 sec 2023-11-12T09:30:23,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-12T09:30:23,436 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: backup:system, procId: 26 completed 2023-11-12T09:30:23,452 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:30:23,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] procedure2.ProcedureExecutor(1032): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system_bulk 2023-11-12T09:30:23,457 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:30:23,457 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:23,457 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2023-11-12T09:30:23,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-12T09:30:23,462 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:30:23,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-12T09:30:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-12T09:30:23,884 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => f85d069e361064154880bd8654ab12c0, NAME => 'backup:system_bulk,,1699781423452.f85d069e361064154880bd8654ab12c0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a 2023-11-12T09:30:24,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-12T09:30:24,296 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1699781423452.f85d069e361064154880bd8654ab12c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:30:24,296 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1668): Closing f85d069e361064154880bd8654ab12c0, disabling compactions & flushes 2023-11-12T09:30:24,296 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1699781423452.f85d069e361064154880bd8654ab12c0. 2023-11-12T09:30:24,296 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1699781423452.f85d069e361064154880bd8654ab12c0. 2023-11-12T09:30:24,297 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1699781423452.f85d069e361064154880bd8654ab12c0. after waiting 1 ms 2023-11-12T09:30:24,297 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1699781423452.f85d069e361064154880bd8654ab12c0. 2023-11-12T09:30:24,297 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1909): Closed backup:system_bulk,,1699781423452.f85d069e361064154880bd8654ab12c0. 2023-11-12T09:30:24,297 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1622): Region close journal for f85d069e361064154880bd8654ab12c0: 2023-11-12T09:30:24,298 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:30:24,298 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system_bulk,,1699781423452.f85d069e361064154880bd8654ab12c0.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1699781424298"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781424298"}]},"ts":"1699781424298"} 2023-11-12T09:30:24,300 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:30:24,301 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:30:24,302 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781424301"}]},"ts":"1699781424301"} 2023-11-12T09:30:24,303 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2023-11-12T09:30:24,307 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=f85d069e361064154880bd8654ab12c0, ASSIGN}] 2023-11-12T09:30:24,310 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=f85d069e361064154880bd8654ab12c0, ASSIGN 2023-11-12T09:30:24,311 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system_bulk, region=f85d069e361064154880bd8654ab12c0, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37573,1699781400943; forceNewPlan=false, retain=false 2023-11-12T09:30:24,463 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=f85d069e361064154880bd8654ab12c0, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:24,465 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE; OpenRegionProcedure f85d069e361064154880bd8654ab12c0, server=jenkins-hbase3.apache.org,37573,1699781400943}] 2023-11-12T09:30:24,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-12T09:30:24,618 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:24,622 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(130): Open backup:system_bulk,,1699781423452.f85d069e361064154880bd8654ab12c0. 2023-11-12T09:30:24,622 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7254): Opening region: {ENCODED => f85d069e361064154880bd8654ab12c0, NAME => 'backup:system_bulk,,1699781423452.f85d069e361064154880bd8654ab12c0.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:30:24,622 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:30:24,623 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk f85d069e361064154880bd8654ab12c0 2023-11-12T09:30:24,623 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1699781423452.f85d069e361064154880bd8654ab12c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:30:24,623 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7296): checking encryption for f85d069e361064154880bd8654ab12c0 2023-11-12T09:30:24,623 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7299): checking classloading for f85d069e361064154880bd8654ab12c0 2023-11-12T09:30:24,625 INFO [StoreOpener-f85d069e361064154880bd8654ab12c0-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region f85d069e361064154880bd8654ab12c0 2023-11-12T09:30:24,627 INFO [StoreOpener-f85d069e361064154880bd8654ab12c0-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f85d069e361064154880bd8654ab12c0 columnFamilyName meta 2023-11-12T09:30:24,627 DEBUG [StoreOpener-f85d069e361064154880bd8654ab12c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:24,627 INFO [StoreOpener-f85d069e361064154880bd8654ab12c0-1 {}] regionserver.HStore(324): Store=f85d069e361064154880bd8654ab12c0/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:30:24,627 INFO [StoreOpener-f85d069e361064154880bd8654ab12c0-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region f85d069e361064154880bd8654ab12c0 2023-11-12T09:30:24,629 INFO [StoreOpener-f85d069e361064154880bd8654ab12c0-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f85d069e361064154880bd8654ab12c0 columnFamilyName session 2023-11-12T09:30:24,629 DEBUG [StoreOpener-f85d069e361064154880bd8654ab12c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:24,629 INFO [StoreOpener-f85d069e361064154880bd8654ab12c0-1 {}] regionserver.HStore(324): Store=f85d069e361064154880bd8654ab12c0/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:30:24,630 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system_bulk/f85d069e361064154880bd8654ab12c0 2023-11-12T09:30:24,631 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system_bulk/f85d069e361064154880bd8654ab12c0 2023-11-12T09:30:24,632 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-11-12T09:30:24,634 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1076): writing seq id for f85d069e361064154880bd8654ab12c0 2023-11-12T09:30:24,637 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system_bulk/f85d069e361064154880bd8654ab12c0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:30:24,638 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): Opened f85d069e361064154880bd8654ab12c0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11947549120, jitterRate=0.11270222067832947}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-11-12T09:30:24,639 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(992): Region open journal for f85d069e361064154880bd8654ab12c0: 2023-11-12T09:30:24,639 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system_bulk,,1699781423452.f85d069e361064154880bd8654ab12c0., pid=31, masterSystemTime=1699781424617 2023-11-12T09:30:24,641 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system_bulk,,1699781423452.f85d069e361064154880bd8654ab12c0. 2023-11-12T09:30:24,641 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(158): Opened backup:system_bulk,,1699781423452.f85d069e361064154880bd8654ab12c0. 2023-11-12T09:30:24,642 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=f85d069e361064154880bd8654ab12c0, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:24,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=31, resume processing ppid=30 2023-11-12T09:30:24,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=31, ppid=30, state=SUCCESS; OpenRegionProcedure f85d069e361064154880bd8654ab12c0, server=jenkins-hbase3.apache.org,37573,1699781400943 in 179 msec 2023-11-12T09:30:24,647 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=30, resume processing ppid=29 2023-11-12T09:30:24,648 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=30, ppid=29, state=SUCCESS; TransitRegionStateProcedure table=backup:system_bulk, region=f85d069e361064154880bd8654ab12c0, ASSIGN in 339 msec 2023-11-12T09:30:24,648 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:30:24,648 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781424648"}]},"ts":"1699781424648"} 2023-11-12T09:30:24,650 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2023-11-12T09:30:24,653 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:30:24,654 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=29, state=SUCCESS; CreateTableProcedure table=backup:system_bulk in 1.2010 sec 2023-11-12T09:30:25,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-12T09:30:25,565 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: backup:system_bulk, procId: 29 completed 2023-11-12T09:30:25,579 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:30:25,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.BinaryComparator in 0ms 2023-11-12T09:30:25,614 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:30:25,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781425614 (current time:1699781425614). 2023-11-12T09:30:25,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:30:25,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:30:25,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x08b6ec40 to 127.0.0.1:57951 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:30:25,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42b299c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:30:25,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:30:25,631 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:49876, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:30:25,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x08b6ec40 to 127.0.0.1:57951 2023-11-12T09:30:25,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:30:25,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:30:25,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] procedure2.ProcedureExecutor(1032): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:30:25,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-11-12T09:30:25,650 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:30:25,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-12T09:30:25,659 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:30:25,675 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/.hbase-snapshot, current working dir path: hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:30:25,678 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:30:25,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-12T09:30:25,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-12T09:30:26,091 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:30:26,094 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 441fc40990b285ba0c11d5ae2272b2af}] 2023-11-12T09:30:26,102 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 441fc40990b285ba0c11d5ae2272b2af 2023-11-12T09:30:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-12T09:30:26,262 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:26,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37573 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2023-11-12T09:30:26,263 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1699781421319.441fc40990b285ba0c11d5ae2272b2af. 2023-11-12T09:30:26,266 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2819): Flushing 441fc40990b285ba0c11d5ae2272b2af 2/2 column families, dataSize=45 B heapSize=632 B 2023-11-12T09:30:26,347 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/.tmp/session/69aeda0ec11340f6be24d8bb52e8f078 is 49, key is activesession:/session:c/1699781425600/Put/seqid=0 2023-11-12T09:30:26,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-12T09:30:26,770 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/.tmp/session/69aeda0ec11340f6be24d8bb52e8f078 2023-11-12T09:30:26,844 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/.tmp/session/69aeda0ec11340f6be24d8bb52e8f078 as hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/session/69aeda0ec11340f6be24d8bb52e8f078 2023-11-12T09:30:26,851 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/session/69aeda0ec11340f6be24d8bb52e8f078, entries=1, sequenceid=5, filesize=4.9 K 2023-11-12T09:30:26,856 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3022): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for 441fc40990b285ba0c11d5ae2272b2af in 590ms, sequenceid=5, compaction requested=false 2023-11-12T09:30:26,856 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-11-12T09:30:26,859 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2520): Flush status journal for 441fc40990b285ba0c11d5ae2272b2af: 2023-11-12T09:30:26,859 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1699781421319.441fc40990b285ba0c11d5ae2272b2af. for snapshot_backup_system completed. 2023-11-12T09:30:26,860 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1699781421319.441fc40990b285ba0c11d5ae2272b2af.' region-info for snapshot=snapshot_backup_system 2023-11-12T09:30:26,867 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:30:26,871 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-11-12T09:30:26,872 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/session/69aeda0ec11340f6be24d8bb52e8f078] hfiles 2023-11-12T09:30:26,872 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/session/69aeda0ec11340f6be24d8bb52e8f078 for snapshot=snapshot_backup_system 2023-11-12T09:30:26,889 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1699781421319.441fc40990b285ba0c11d5ae2272b2af. 2023-11-12T09:30:26,892 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=33 2023-11-12T09:30:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.HMaster(4026): Remote procedure done, pid=33 2023-11-12T09:30:26,896 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 441fc40990b285ba0c11d5ae2272b2af 2023-11-12T09:30:26,897 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 441fc40990b285ba0c11d5ae2272b2af 2023-11-12T09:30:26,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=33, resume processing ppid=32 2023-11-12T09:30:26,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=33, ppid=32, state=SUCCESS; SnapshotRegionProcedure 441fc40990b285ba0c11d5ae2272b2af in 804 msec 2023-11-12T09:30:26,900 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:30:26,902 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:30:26,905 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:30:26,905 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-12T09:30:26,908 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-12T09:30:27,008 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2023-11-12T09:30:27,338 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:30:27,351 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:30:27,352 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/.hbase-snapshot/snapshot_backup_system 2023-11-12T09:30:27,354 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:30:27,354 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-11-12T09:30:27,355 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=32, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.7120 sec 2023-11-12T09:30:27,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-12T09:30:27,763 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: backup:system, procId: 32 completed 2023-11-12T09:30:27,764 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] impl.TableBackupClient(115): Backup backup_1699781421145 started at 1699781427763. 2023-11-12T09:30:27,786 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.TableBackupClient(119): Backup session backup_1699781421145 has been started. 2023-11-12T09:30:27,789 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-12T09:30:27,800 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(962): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-11-12T09:30:27,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-12T09:30:27,804 INFO [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-12T09:30:27,804 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-12T09:30:27,806 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-12T09:30:27,806 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-12T09:30:27,806 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-12T09:30:27,806 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:30:27,810 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-12T09:30:27,810 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:27,810 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:30:27,810 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:30:27,810 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:27,810 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-12T09:30:27,811 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-12T09:30:27,811 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-12T09:30:27,812 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-12T09:30:27,812 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:30:27,812 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-12T09:30:27,814 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-12T09:30:27,814 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-12T09:30:27,814 DEBUG [member: 'jenkins-hbase3.apache.org,37573,1699781400943' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-12T09:30:27,814 DEBUG [member: 'jenkins-hbase3.apache.org,37573,1699781400943' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-12T09:30:27,815 DEBUG [member: 'jenkins-hbase3.apache.org,37573,1699781400943' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-12T09:30:27,815 DEBUG [member: 'jenkins-hbase3.apache.org,37573,1699781400943' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-12T09:30:27,815 DEBUG [member: 'jenkins-hbase3.apache.org,37573,1699781400943' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,37573,1699781400943' joining acquired barrier for procedure (rolllog) in zk 2023-11-12T09:30:27,818 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:27,818 DEBUG [member: 'jenkins-hbase3.apache.org,37573,1699781400943' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-12T09:30:27,818 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:27,818 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:30:27,818 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:30:27,819 DEBUG [member: 'jenkins-hbase3.apache.org,37573,1699781400943' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-11-12T09:30:27,819 DEBUG [member: 'jenkins-hbase3.apache.org,37573,1699781400943' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-12T09:30:27,819 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:30:27,819 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:30:27,819 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:30:27,820 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:27,820 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:30:27,821 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,37573,1699781400943' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-12T09:30:27,821 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@3c5fd5c2[Count = 0] remaining members to acquire global barrier 2023-11-12T09:30:27,821 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-12T09:30:27,821 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-12T09:30:27,822 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:30:27,822 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-12T09:30:27,822 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-12T09:30:27,822 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:27,822 DEBUG [member: 'jenkins-hbase3.apache.org,37573,1699781400943' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-12T09:30:27,822 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-12T09:30:27,824 DEBUG [member: 'jenkins-hbase3.apache.org,37573,1699781400943' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-12T09:30:27,824 DEBUG [rs(jenkins-hbase3.apache.org,37573,1699781400943)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:27,824 INFO [rs(jenkins-hbase3.apache.org,37573,1699781400943)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1699781402686 highest: 1699781402686 on jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:27,824 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37573%2C1699781400943.meta:.meta(num 1699781403220) roll requested 2023-11-12T09:30:27,835 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/WALs/jenkins-hbase3.apache.org,37573,1699781400943/jenkins-hbase3.apache.org%2C37573%2C1699781400943.meta.1699781427824.meta, exclude list is [], retry=0 2023-11-12T09:30:27,843 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44263,DS-4bdce35f-16ea-409e-9d44-e8db6ed957eb,DISK] 2023-11-12T09:30:27,847 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/WALs/jenkins-hbase3.apache.org,37573,1699781400943/jenkins-hbase3.apache.org%2C37573%2C1699781400943.meta.1699781403220.meta with entries=36, filesize=11.13 KB; new WAL /user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/WALs/jenkins-hbase3.apache.org,37573,1699781400943/jenkins-hbase3.apache.org%2C37573%2C1699781400943.meta.1699781427824.meta 2023-11-12T09:30:27,848 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:44263,DS-4bdce35f-16ea-409e-9d44-e8db6ed957eb,DISK]] 2023-11-12T09:30:27,848 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/WALs/jenkins-hbase3.apache.org,37573,1699781400943/jenkins-hbase3.apache.org%2C37573%2C1699781400943.meta.1699781403220.meta is not closed yet, will try archiving it next time 2023-11-12T09:30:27,848 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37573%2C1699781400943:(num 1699781402686) roll requested 2023-11-12T09:30:27,861 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/WALs/jenkins-hbase3.apache.org,37573,1699781400943/jenkins-hbase3.apache.org%2C37573%2C1699781400943.1699781427849, exclude list is [], retry=0 2023-11-12T09:30:27,865 DEBUG [rs(jenkins-hbase3.apache.org,37573,1699781400943)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 41 2023-11-12T09:30:27,865 INFO [rs(jenkins-hbase3.apache.org,37573,1699781400943)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1699781427849 on jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:27,867 DEBUG [rs(jenkins-hbase3.apache.org,37573,1699781400943)-backup-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:30:27,867 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44263,DS-4bdce35f-16ea-409e-9d44-e8db6ed957eb,DISK] 2023-11-12T09:30:27,869 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:49074, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-11-12T09:30:27,872 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/WALs/jenkins-hbase3.apache.org,37573,1699781400943/jenkins-hbase3.apache.org%2C37573%2C1699781400943.1699781402686 with entries=19, filesize=3.99 KB; new WAL /user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/WALs/jenkins-hbase3.apache.org,37573,1699781400943/jenkins-hbase3.apache.org%2C37573%2C1699781400943.1699781427849 2023-11-12T09:30:27,872 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:44263,DS-4bdce35f-16ea-409e-9d44-e8db6ed957eb,DISK]] 2023-11-12T09:30:27,872 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/WALs/jenkins-hbase3.apache.org,37573,1699781400943/jenkins-hbase3.apache.org%2C37573%2C1699781400943.1699781402686 is not closed yet, will try archiving it next time 2023-11-12T09:30:27,898 DEBUG [member: 'jenkins-hbase3.apache.org,37573,1699781400943' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-12T09:30:27,898 DEBUG [member: 'jenkins-hbase3.apache.org,37573,1699781400943' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,37573,1699781400943' in zk 2023-11-12T09:30:27,900 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:27,900 DEBUG [member: 'jenkins-hbase3.apache.org,37573,1699781400943' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-12T09:30:27,900 DEBUG [member: 'jenkins-hbase3.apache.org,37573,1699781400943' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:30:27,900 DEBUG [member: 'jenkins-hbase3.apache.org,37573,1699781400943' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-12T09:30:27,900 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:27,902 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:30:27,902 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:30:27,904 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:30:27,905 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:30:27,905 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:30:27,906 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:27,906 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:30:27,907 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:30:27,907 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:27,908 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,37573,1699781400943': 2023-11-12T09:30:27,908 INFO [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-12T09:30:27,908 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,37573,1699781400943' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-12T09:30:27,908 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-12T09:30:27,908 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-12T09:30:27,908 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-12T09:30:27,908 INFO [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-12T09:30:27,910 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:30:27,910 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:30:27,910 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-12T09:30:27,911 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-12T09:30:27,911 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-12T09:30:27,911 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:30:27,911 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:30:27,911 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:30:27,911 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:30:27,911 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:30:27,911 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:30:27,912 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:30:27,912 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:27,912 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:30:27,912 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:30:27,913 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:30:27,913 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:27,913 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:30:27,914 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:27,914 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:30:27,914 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:27,924 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:27,924 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-12T09:30:27,924 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-12T09:30:27,925 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-12T09:30:27,925 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:27,925 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:30:27,925 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:30:27,925 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:30:27,925 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:30:27,925 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:30:27,925 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-12T09:30:27,925 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:30:27,926 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-12T09:30:27,927 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-12T09:30:27,927 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:30:27,927 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:30:27,929 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] client.HBaseAdmin(2906): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-11-12T09:30:27,929 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] client.HBaseAdmin(2914): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-11-12T09:30:28,002 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:30:28,029 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] client.HBaseAdmin(2920): Getting current status of procedure from master... 2023-11-12T09:30:28,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-12T09:30:28,047 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1699781428047_default_test-1699781414320 table=test-1699781414320 type=FLUSH ttl=0 } 2023-11-12T09:30:28,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781428047 (current time:1699781428047). 2023-11-12T09:30:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:30:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1699781428047_default_test-1699781414320 VERSION not specified, setting to 2 2023-11-12T09:30:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x30a1eb2c to 127.0.0.1:57951 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:30:28,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59183afd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:30:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:30:28,069 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:49890, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:30:28,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x30a1eb2c to 127.0.0.1:57951 2023-11-12T09:30:28,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:30:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:30:28,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] procedure2.ProcedureExecutor(1032): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781428047_default_test-1699781414320 table=test-1699781414320 type=FLUSH ttl=0 } 2023-11-12T09:30:28,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1699781428047_default_test-1699781414320 table=test-1699781414320 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-11-12T09:30:28,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-12T09:30:28,075 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781428047_default_test-1699781414320 table=test-1699781414320 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:30:28,076 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781428047_default_test-1699781414320 table=test-1699781414320 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:30:28,084 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/.hbase-snapshot, current working dir path: hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/.hbase-snapshot/.tmp/snapshot_1699781428047_default_test-1699781414320 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:30:28,086 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781428047_default_test-1699781414320 table=test-1699781414320 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:30:28,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-12T09:30:28,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-12T09:30:28,503 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781428047_default_test-1699781414320 table=test-1699781414320 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:30:28,503 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 37c7f0bb858c8098fd397daa8ed9cc98}] 2023-11-12T09:30:28,505 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 37c7f0bb858c8098fd397daa8ed9cc98 2023-11-12T09:30:28,656 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:28,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37573 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2023-11-12T09:30:28,658 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1699781414320,,1699781414853.37c7f0bb858c8098fd397daa8ed9cc98. 2023-11-12T09:30:28,659 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2819): Flushing 37c7f0bb858c8098fd397daa8ed9cc98 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-11-12T09:30:28,677 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/test-1699781414320/37c7f0bb858c8098fd397daa8ed9cc98/.tmp/f/0502255d783947ce89e78b17fc613b51 is 37, key is row10/f:q1/1699781417030/Put/seqid=0 2023-11-12T09:30:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-12T09:30:28,915 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-11-12T09:30:28,915 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-11-12T09:30:28,916 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-11-12T09:30:28,916 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2023-11-12T09:30:28,916 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16997814143203 2023-11-12T09:30:28,917 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-16997814143203 Metrics about Tables on a single HBase RegionServer 2023-11-12T09:30:29,088 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/test-1699781414320/37c7f0bb858c8098fd397daa8ed9cc98/.tmp/f/0502255d783947ce89e78b17fc613b51 2023-11-12T09:30:29,098 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/test-1699781414320/37c7f0bb858c8098fd397daa8ed9cc98/.tmp/f/0502255d783947ce89e78b17fc613b51 as hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/test-1699781414320/37c7f0bb858c8098fd397daa8ed9cc98/f/0502255d783947ce89e78b17fc613b51 2023-11-12T09:30:29,105 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/test-1699781414320/37c7f0bb858c8098fd397daa8ed9cc98/f/0502255d783947ce89e78b17fc613b51, entries=99, sequenceid=103, filesize=8.2 K 2023-11-12T09:30:29,107 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 37c7f0bb858c8098fd397daa8ed9cc98 in 448ms, sequenceid=103, compaction requested=false 2023-11-12T09:30:29,108 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2520): Flush status journal for 37c7f0bb858c8098fd397daa8ed9cc98: 2023-11-12T09:30:29,108 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1699781414320,,1699781414853.37c7f0bb858c8098fd397daa8ed9cc98. for snapshot_1699781428047_default_test-1699781414320 completed. 2023-11-12T09:30:29,108 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(240): Storing 'test-1699781414320,,1699781414853.37c7f0bb858c8098fd397daa8ed9cc98.' region-info for snapshot=snapshot_1699781428047_default_test-1699781414320 2023-11-12T09:30:29,109 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:30:29,109 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/test-1699781414320/37c7f0bb858c8098fd397daa8ed9cc98/f/0502255d783947ce89e78b17fc613b51] hfiles 2023-11-12T09:30:29,109 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/test-1699781414320/37c7f0bb858c8098fd397daa8ed9cc98/f/0502255d783947ce89e78b17fc613b51 for snapshot=snapshot_1699781428047_default_test-1699781414320 2023-11-12T09:30:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-12T09:30:29,521 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1699781414320,,1699781414853.37c7f0bb858c8098fd397daa8ed9cc98. 2023-11-12T09:30:29,521 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=35 2023-11-12T09:30:29,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.HMaster(4026): Remote procedure done, pid=35 2023-11-12T09:30:29,522 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1699781428047_default_test-1699781414320 on region 37c7f0bb858c8098fd397daa8ed9cc98 2023-11-12T09:30:29,523 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 37c7f0bb858c8098fd397daa8ed9cc98 2023-11-12T09:30:29,526 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=35, resume processing ppid=34 2023-11-12T09:30:29,526 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781428047_default_test-1699781414320 table=test-1699781414320 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:30:29,527 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=35, ppid=34, state=SUCCESS; SnapshotRegionProcedure 37c7f0bb858c8098fd397daa8ed9cc98 in 1.0220 sec 2023-11-12T09:30:29,527 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781428047_default_test-1699781414320 table=test-1699781414320 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:30:29,528 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781428047_default_test-1699781414320 table=test-1699781414320 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:30:29,528 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1699781428047_default_test-1699781414320 2023-11-12T09:30:29,529 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/.hbase-snapshot/.tmp/snapshot_1699781428047_default_test-1699781414320 2023-11-12T09:30:29,945 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781428047_default_test-1699781414320 table=test-1699781414320 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:30:29,953 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781428047_default_test-1699781414320 table=test-1699781414320 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:30:29,953 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/.hbase-snapshot/.tmp/snapshot_1699781428047_default_test-1699781414320 to hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/.hbase-snapshot/snapshot_1699781428047_default_test-1699781414320 2023-11-12T09:30:29,955 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781428047_default_test-1699781414320 table=test-1699781414320 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:30:29,955 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1699781428047_default_test-1699781414320 table=test-1699781414320 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-11-12T09:30:29,956 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=34, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781428047_default_test-1699781414320 table=test-1699781414320 type=FLUSH ttl=0 } in 1.8840 sec 2023-11-12T09:30:30,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-12T09:30:30,183 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: default:test-1699781414320, procId: 34 completed 2023-11-12T09:30:30,183 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1699781421145 2023-11-12T09:30:30,183 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-12T09:30:30,185 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-11-12T09:30:30,187 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1699781421145_test-1699781414320 2023-11-12T09:30:30,187 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1699781428047_default_test-1699781414320 to hdfs://localhost:41663/backupUT/backup_1699781421145/default/test-1699781414320/ 2023-11-12T09:30:30,197 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-12T09:30:30,226 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:38625, inputRoot=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a 2023-11-12T09:30:30,226 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,44289,1699781397811_238163138_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:41663/backupUT/backup_1699781421145/default/test-1699781414320, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:41663/backupUT/backup_1699781421145/default/test-1699781414320/.hbase-snapshot/.tmp/snapshot_1699781428047_default_test-1699781414320 2023-11-12T09:30:30,227 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:38625, inputRoot=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a, snapshotDir=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/.hbase-snapshot/snapshot_1699781428047_default_test-1699781414320. 2023-11-12T09:30:30,237 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/.hbase-snapshot/snapshot_1699781428047_default_test-1699781414320 to hdfs://localhost:41663/backupUT/backup_1699781421145/default/test-1699781414320/.hbase-snapshot/.tmp/snapshot_1699781428047_default_test-1699781414320 2023-11-12T09:30:30,678 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:30,678 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:30,679 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:30,679 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:31,674 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/hadoop-5449863035759025125.jar 2023-11-12T09:30:31,674 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:31,675 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:31,675 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:31,676 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:31,676 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:31,677 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:31,677 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:31,677 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:31,678 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:31,678 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:30:31,679 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:30:31,680 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:30:31,680 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:30:31,680 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:30:31,681 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:30:31,681 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:30:31,682 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:30:31,682 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:30:31,683 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:30:31,683 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:30:31,684 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:30:31,688 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:30:31,689 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:30:31,689 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:30:31,690 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:30:31,690 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:30:31,690 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:30:31,691 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:30:31,750 WARN [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-12T09:30:31,776 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1699781428047_default_test-1699781414320' hfile list 2023-11-12T09:30:31,786 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-11-12T09:30:32,028 WARN [Listener at jenkins-hbase3.apache.org/40129 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/mapreduce.cluster.local.dir/1699781431974/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:30:32,029 WARN [Listener at jenkins-hbase3.apache.org/40129 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/mapreduce.cluster.local.dir/1699781431974/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:30:32,188 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-12T09:30:32,248 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/test-1699781414320/37c7f0bb858c8098fd397daa8ed9cc98/f/0502255d783947ce89e78b17fc613b51 output=hdfs://localhost:41663/backupUT/backup_1699781421145/default/test-1699781414320/archive/data/default/test-1699781414320/37c7f0bb858c8098fd397daa8ed9cc98/f/0502255d783947ce89e78b17fc613b51 2023-11-12T09:30:32,248 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 8.007M/sec 2023-11-12T09:30:33,092 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-12T09:30:33,093 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-12T09:30:33,100 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1699781428047_default_test-1699781414320 2023-11-12T09:30:33,100 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1699781428047_default_test-1699781414320 finished. 2023-11-12T09:30:33,101 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.BackupSystemTable(1034): test-1699781414320 2023-11-12T09:30:33,172 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781421145 2023-11-12T09:30:33,172 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-12T09:30:33,596 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:41663/backupUT/backup_1699781421145/.backup.manifest 2023-11-12T09:30:33,596 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.TableBackupClient(358): Backup backup_1699781421145 finished: type=FULL,tablelist=test-1699781414320,targetRootDir=hdfs://localhost:41663/backupUT,startts=1699781427763,completets=1699781433170,bytescopied=0 2023-11-12T09:30:33,596 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-12T09:30:33,596 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1699781428047_default_test-1699781414320 2023-11-12T09:30:33,603 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_1699781428047_default_test-1699781414320" 2023-11-12T09:30:33,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1699781428047_default_test-1699781414320 2023-11-12T09:30:33,609 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1699781428047_default_test-1699781414320 for backup backup_1699781421145 succeeded. 2023-11-12T09:30:33,609 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:30:33,622 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:30:33,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-12T09:30:33,625 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-12T09:30:33,635 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:30:33,655 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] impl.TableBackupClient(376): Backup backup_1699781421145 completed. 2023-11-12T09:30:33,655 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-12T09:30:33,655 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x708a33bc to 127.0.0.1:57951 2023-11-12T09:30:33,655 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:30:33,657 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] backup.TestRemoteRestore(73): backup complete 2023-11-12T09:30:33,660 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] impl.RestoreTablesClient(101): HBase table table1 does not exist. It will be created during restore process 2023-11-12T09:30:33,661 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41663/backupUT/backup_1699781421145/.backup.manifest 2023-11-12T09:30:33,664 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781421145/.backup.manifest 2023-11-12T09:30:33,670 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41663/backupUT/backup_1699781421145/.backup.manifest 2023-11-12T09:30:33,672 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781421145/.backup.manifest 2023-11-12T09:30:33,672 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] impl.RestoreTablesClient(148): Restoring 'test-1699781414320' to 'table1' from full backup image hdfs://localhost:41663/backupUT/backup_1699781421145/default/test-1699781414320 2023-11-12T09:30:33,681 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] util.RestoreTool(487): Creating target table 'table1' 2023-11-12T09:30:33,681 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:41663/backupUT/backup_1699781421145/default/test-1699781414320/archive/data/default/test-1699781414320/37c7f0bb858c8098fd397daa8ed9cc98 2023-11-12T09:30:33,682 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:41663/backupUT/backup_1699781421145/default/test-1699781414320/archive/data/default/test-1699781414320/37c7f0bb858c8098fd397daa8ed9cc98/f in region [hdfs://localhost:41663/backupUT/backup_1699781421145/default/test-1699781414320/archive/data/default/test-1699781414320/37c7f0bb858c8098fd397daa8ed9cc98] 2023-11-12T09:30:33,687 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] util.RestoreTool(442): Trying to figure out region boundaries hfile=hdfs://localhost:41663/backupUT/backup_1699781421145/default/test-1699781414320/archive/data/default/test-1699781414320/37c7f0bb858c8098fd397daa8ed9cc98/f/0502255d783947ce89e78b17fc613b51 first=row0 last=row98 2023-11-12T09:30:33,695 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:30:33,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] procedure2.ProcedureExecutor(1032): Stored pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=table1 2023-11-12T09:30:33,698 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:30:33,698 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 36 2023-11-12T09:30:33,699 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:30:33,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-12T09:30:33,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-12T09:30:34,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-12T09:30:34,108 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 5bdf91d1fb4f5b624b93a3ddc48685df, NAME => 'table1,,1699781433694.5bdf91d1fb4f5b624b93a3ddc48685df.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a 2023-11-12T09:30:34,161 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-11-12T09:30:34,163 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:49088, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-11-12T09:30:34,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-12T09:30:34,515 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(888): Instantiated table1,,1699781433694.5bdf91d1fb4f5b624b93a3ddc48685df.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:30:34,515 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1668): Closing 5bdf91d1fb4f5b624b93a3ddc48685df, disabling compactions & flushes 2023-11-12T09:30:34,515 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1690): Closing region table1,,1699781433694.5bdf91d1fb4f5b624b93a3ddc48685df. 2023-11-12T09:30:34,515 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1699781433694.5bdf91d1fb4f5b624b93a3ddc48685df. 2023-11-12T09:30:34,515 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on table1,,1699781433694.5bdf91d1fb4f5b624b93a3ddc48685df. after waiting 0 ms 2023-11-12T09:30:34,515 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region table1,,1699781433694.5bdf91d1fb4f5b624b93a3ddc48685df. 2023-11-12T09:30:34,515 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1909): Closed table1,,1699781433694.5bdf91d1fb4f5b624b93a3ddc48685df. 2023-11-12T09:30:34,515 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1622): Region close journal for 5bdf91d1fb4f5b624b93a3ddc48685df: 2023-11-12T09:30:34,517 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:30:34,517 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"table1,,1699781433694.5bdf91d1fb4f5b624b93a3ddc48685df.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1699781434517"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781434517"}]},"ts":"1699781434517"} 2023-11-12T09:30:34,519 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:30:34,520 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:30:34,520 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781434520"}]},"ts":"1699781434520"} 2023-11-12T09:30:34,522 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLING in hbase:meta 2023-11-12T09:30:34,526 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=5bdf91d1fb4f5b624b93a3ddc48685df, ASSIGN}] 2023-11-12T09:30:34,528 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=5bdf91d1fb4f5b624b93a3ddc48685df, ASSIGN 2023-11-12T09:30:34,529 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=table1, region=5bdf91d1fb4f5b624b93a3ddc48685df, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37573,1699781400943; forceNewPlan=false, retain=false 2023-11-12T09:30:34,681 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=5bdf91d1fb4f5b624b93a3ddc48685df, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:34,683 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE; OpenRegionProcedure 5bdf91d1fb4f5b624b93a3ddc48685df, server=jenkins-hbase3.apache.org,37573,1699781400943}] 2023-11-12T09:30:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-12T09:30:34,835 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:34,839 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(130): Open table1,,1699781433694.5bdf91d1fb4f5b624b93a3ddc48685df. 2023-11-12T09:30:34,839 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7254): Opening region: {ENCODED => 5bdf91d1fb4f5b624b93a3ddc48685df, NAME => 'table1,,1699781433694.5bdf91d1fb4f5b624b93a3ddc48685df.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:30:34,840 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:30:34,840 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 5bdf91d1fb4f5b624b93a3ddc48685df 2023-11-12T09:30:34,840 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(888): Instantiated table1,,1699781433694.5bdf91d1fb4f5b624b93a3ddc48685df.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:30:34,840 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7296): checking encryption for 5bdf91d1fb4f5b624b93a3ddc48685df 2023-11-12T09:30:34,840 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7299): checking classloading for 5bdf91d1fb4f5b624b93a3ddc48685df 2023-11-12T09:30:34,842 INFO [StoreOpener-5bdf91d1fb4f5b624b93a3ddc48685df-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 5bdf91d1fb4f5b624b93a3ddc48685df 2023-11-12T09:30:34,843 INFO [StoreOpener-5bdf91d1fb4f5b624b93a3ddc48685df-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bdf91d1fb4f5b624b93a3ddc48685df columnFamilyName f 2023-11-12T09:30:34,843 DEBUG [StoreOpener-5bdf91d1fb4f5b624b93a3ddc48685df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:34,844 INFO [StoreOpener-5bdf91d1fb4f5b624b93a3ddc48685df-1 {}] regionserver.HStore(324): Store=5bdf91d1fb4f5b624b93a3ddc48685df/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:30:34,845 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/table1/5bdf91d1fb4f5b624b93a3ddc48685df 2023-11-12T09:30:34,845 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/table1/5bdf91d1fb4f5b624b93a3ddc48685df 2023-11-12T09:30:34,848 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1076): writing seq id for 5bdf91d1fb4f5b624b93a3ddc48685df 2023-11-12T09:30:34,850 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/table1/5bdf91d1fb4f5b624b93a3ddc48685df/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:30:34,851 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): Opened 5bdf91d1fb4f5b624b93a3ddc48685df; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11737038720, jitterRate=0.09309691190719604}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:30:34,851 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(992): Region open journal for 5bdf91d1fb4f5b624b93a3ddc48685df: 2023-11-12T09:30:34,852 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2556): Post open deploy tasks for table1,,1699781433694.5bdf91d1fb4f5b624b93a3ddc48685df., pid=38, masterSystemTime=1699781434835 2023-11-12T09:30:34,854 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2583): Finished post open deploy task for table1,,1699781433694.5bdf91d1fb4f5b624b93a3ddc48685df. 2023-11-12T09:30:34,854 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(158): Opened table1,,1699781433694.5bdf91d1fb4f5b624b93a3ddc48685df. 2023-11-12T09:30:34,855 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=5bdf91d1fb4f5b624b93a3ddc48685df, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:34,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=38, resume processing ppid=37 2023-11-12T09:30:34,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=38, ppid=37, state=SUCCESS; OpenRegionProcedure 5bdf91d1fb4f5b624b93a3ddc48685df, server=jenkins-hbase3.apache.org,37573,1699781400943 in 173 msec 2023-11-12T09:30:34,860 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=37, resume processing ppid=36 2023-11-12T09:30:34,860 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=37, ppid=36, state=SUCCESS; TransitRegionStateProcedure table=table1, region=5bdf91d1fb4f5b624b93a3ddc48685df, ASSIGN in 332 msec 2023-11-12T09:30:34,861 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:30:34,861 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781434861"}]},"ts":"1699781434861"} 2023-11-12T09:30:34,862 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLED in hbase:meta 2023-11-12T09:30:34,866 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:30:34,867 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=36, state=SUCCESS; CreateTableProcedure table=table1 in 1.1710 sec 2023-11-12T09:30:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-12T09:30:35,807 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: default:table1, procId: 36 completed 2023-11-12T09:30:35,812 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:41663/backupUT/backup_1699781421145/default/test-1699781414320/archive/data/default/test-1699781414320/37c7f0bb858c8098fd397daa8ed9cc98 from hbase tables test-1699781414320 to tables table1 2023-11-12T09:30:35,812 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.MapReduceRestoreJob(72): Restore test-1699781414320 into table1 2023-11-12T09:30:35,815 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/hbase-staging/bulk_output-default-table1-1699781435812 from hdfs://localhost:41663/backupUT/backup_1699781421145/default/test-1699781414320/archive/data/default/test-1699781414320/37c7f0bb858c8098fd397daa8ed9cc98 2023-11-12T09:30:35,819 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0d7ddc68 to 127.0.0.1:57951 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:30:35,825 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fb9b09b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:30:35,826 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:30:35,827 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:46510, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:30:35,842 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-11-12T09:30:35,842 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table table1 2023-11-12T09:30:35,848 DEBUG [hconnection-0x797b6618-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:30:35,850 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:60300, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:30:35,854 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for table1: 2023-11-12T09:30:35,855 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-11-12T09:30:35,856 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/hbase-staging/partitions_31e7f7bc-e223-4170-a121-1c53104d6576 2023-11-12T09:30:36,302 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:36,303 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:36,303 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:36,304 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:37,293 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/hadoop-2692648161911818451.jar 2023-11-12T09:30:37,294 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:37,294 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:37,295 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:37,295 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:37,296 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:37,296 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:37,296 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:37,297 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:37,297 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:37,298 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:30:37,299 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:30:37,299 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:30:37,299 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:30:37,300 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:30:37,301 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:30:37,302 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:30:37,302 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:30:37,303 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:30:37,303 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:30:37,304 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:30:37,304 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:30:37,308 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:37,308 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:37,309 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:37,309 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:37,309 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:37,309 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:37,310 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:30:37,310 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: table1 2023-11-12T09:30:37,310 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-12T09:30:37,311 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0d7ddc68 to 127.0.0.1:57951 2023-11-12T09:30:37,311 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:30:37,311 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-11-12T09:30:37,311 WARN [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-11-12T09:30:37,312 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:30:37,319 WARN [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-12T09:30:37,326 WARN [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-11-12T09:30:37,417 WARN [Listener at jenkins-hbase3.apache.org/40129 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/mapreduce.cluster.local.dir/1699781437377/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:30:37,417 WARN [Listener at jenkins-hbase3.apache.org/40129 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/mapreduce.cluster.local.dir/1699781437377/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:30:37,494 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:41663/backupUT/backup_1699781421145/default/test-1699781414320/archive/data/default/test-1699781414320/37c7f0bb858c8098fd397daa8ed9cc98/f/0502255d783947ce89e78b17fc613b51 2023-11-12T09:30:37,500 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-11-12T09:30:37,641 DEBUG [pool-323-thread-1 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x48857fef to 127.0.0.1:57951 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:30:37,652 DEBUG [pool-323-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62cea968, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:30:37,654 DEBUG [hconnection-0x11413e68-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:30:37,656 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:60308, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:30:37,658 DEBUG [pool-323-thread-1 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x48857fef to 127.0.0.1:57951 2023-11-12T09:30:37,658 DEBUG [pool-323-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:30:37,661 DEBUG [pool-323-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-11-12T09:30:37,661 DEBUG [pool-323-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-11-12T09:30:37,681 DEBUG [pool-323-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/hbase-staging/bulk_output-default-table1-1699781435812/_temporary/0/_temporary/attempt_local61011903_0002_r_000000_0/f/3b6cc4b0d741407ebd64f8cb4887fd6b is 37, key is row10/f:q1/1699781417030/Put/seqid=0 2023-11-12T09:30:37,684 WARN [Thread-969 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:37573] Specified favored nodes: [jenkins-hbase3.apache.org:37573] 2023-11-12T09:30:38,288 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-11-12T09:30:38,291 INFO [RS-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:51520, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2023-11-12T09:30:38,478 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/hbase-staging/bulk_output-default-table1-1699781435812 2023-11-12T09:30:38,480 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3aa7a75a to 127.0.0.1:57951 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:30:38,487 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@309f6c75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:30:38,488 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:30:38,491 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:60314, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:30:38,501 WARN [Listener at jenkins-hbase3.apache.org/40129 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/hbase-staging/bulk_output-default-table1-1699781435812/_SUCCESS 2023-11-12T09:30:38,507 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:30:38,509 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:46522, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:30:38,543 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/hbase-staging/bulk_output-default-table1-1699781435812/f/3b6cc4b0d741407ebd64f8cb4887fd6b first=Optional[row0] last=Optional[row98] 2023-11-12T09:30:38,572 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=table1,,1699781433694.5bdf91d1fb4f5b624b93a3ddc48685df., hostname=jenkins-hbase3.apache.org,37573,1699781400943, seqNum=2 for row with hfile group [{f,hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/hbase-staging/bulk_output-default-table1-1699781435812/f/3b6cc4b0d741407ebd64f8cb4887fd6b}] 2023-11-12T09:30:38,595 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/hbase-staging/bulk_output-default-table1-1699781435812/f/3b6cc4b0d741407ebd64f8cb4887fd6b for inclusion in 5bdf91d1fb4f5b624b93a3ddc48685df/f 2023-11-12T09:30:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-11-12T09:30:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] regionserver.HStore(625): Region bounds: first= last= 2023-11-12T09:30:38,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] regionserver.HRegion(2520): Flush status journal for 5bdf91d1fb4f5b624b93a3ddc48685df: 2023-11-12T09:30:38,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/hbase-staging/bulk_output-default-table1-1699781435812/f/3b6cc4b0d741407ebd64f8cb4887fd6b to hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/staging/jenkins__table1__jnvfaijrg69r8rj7ll5jao5ts1mis1v61u1td78hmbrd4qisf0vd1dctvdmdkjf8/f/3b6cc4b0d741407ebd64f8cb4887fd6b 2023-11-12T09:30:38,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/staging/jenkins__table1__jnvfaijrg69r8rj7ll5jao5ts1mis1v61u1td78hmbrd4qisf0vd1dctvdmdkjf8/f/3b6cc4b0d741407ebd64f8cb4887fd6b as hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/table1/5bdf91d1fb4f5b624b93a3ddc48685df/f/d9277234f34f4479bea6840d3ef9d267_SeqId_4_ 2023-11-12T09:30:38,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x13a3ad49 to 127.0.0.1:57951 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:30:38,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e3a2aab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:30:38,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:30:38,624 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:46538, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:30:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:30:38,630 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:60316, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:30:38,652 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-12T09:30:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x13a3ad49 to 127.0.0.1:57951 2023-11-12T09:30:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:30:38,653 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/staging/jenkins__table1__jnvfaijrg69r8rj7ll5jao5ts1mis1v61u1td78hmbrd4qisf0vd1dctvdmdkjf8/f/3b6cc4b0d741407ebd64f8cb4887fd6b into 5bdf91d1fb4f5b624b93a3ddc48685df/f as hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/table1/5bdf91d1fb4f5b624b93a3ddc48685df/f/d9277234f34f4479bea6840d3ef9d267_SeqId_4_ - updating store file list. 2023-11-12T09:30:38,662 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/table1/5bdf91d1fb4f5b624b93a3ddc48685df/f/d9277234f34f4479bea6840d3ef9d267_SeqId_4_ into 5bdf91d1fb4f5b624b93a3ddc48685df/f 2023-11-12T09:30:38,663 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/staging/jenkins__table1__jnvfaijrg69r8rj7ll5jao5ts1mis1v61u1td78hmbrd4qisf0vd1dctvdmdkjf8/f/3b6cc4b0d741407ebd64f8cb4887fd6b into 5bdf91d1fb4f5b624b93a3ddc48685df/f (new location: hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/table1/5bdf91d1fb4f5b624b93a3ddc48685df/f/d9277234f34f4479bea6840d3ef9d267_SeqId_4_) 2023-11-12T09:30:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/staging/jenkins__table1__jnvfaijrg69r8rj7ll5jao5ts1mis1v61u1td78hmbrd4qisf0vd1dctvdmdkjf8/f/3b6cc4b0d741407ebd64f8cb4887fd6b 2023-11-12T09:30:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x36784c9a to 127.0.0.1:57951 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:30:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@111a4127, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:30:38,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:30:38,695 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:46552, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-11-12T09:30:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:30:38,702 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:60328, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-11-12T09:30:38,719 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-12T09:30:38,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x36784c9a to 127.0.0.1:57951 2023-11-12T09:30:38,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:30:38,736 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-12T09:30:38,736 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3aa7a75a to 127.0.0.1:57951 2023-11-12T09:30:38,736 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:30:38,737 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-11-12T09:30:38,737 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-11-12T09:30:38,737 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] impl.RestoreTablesClient(233): Backup: backup_1699781421145 hdfs://localhost:41663/backupUT/backup_1699781421145/default/test-1699781414320/ 2023-11-12T09:30:38,737 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-11-12T09:30:38,741 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] client.HBaseAdmin$18(967): Started disable of table1 2023-11-12T09:30:38,748 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.HMaster$13(2695): Client=jenkins//172.31.12.81 disable table1 2023-11-12T09:30:38,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] procedure2.ProcedureExecutor(1032): Stored pid=39, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=table1 2023-11-12T09:30:38,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=39 2023-11-12T09:30:38,767 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781438767"}]},"ts":"1699781438767"} 2023-11-12T09:30:38,770 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLING in hbase:meta 2023-11-12T09:30:38,775 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(293): Set table1 to state=DISABLING 2023-11-12T09:30:38,777 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=5bdf91d1fb4f5b624b93a3ddc48685df, UNASSIGN}] 2023-11-12T09:30:38,780 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=5bdf91d1fb4f5b624b93a3ddc48685df, UNASSIGN 2023-11-12T09:30:38,781 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=5bdf91d1fb4f5b624b93a3ddc48685df, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:38,783 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE; CloseRegionProcedure 5bdf91d1fb4f5b624b93a3ddc48685df, server=jenkins-hbase3.apache.org,37573,1699781400943}] 2023-11-12T09:30:38,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=39 2023-11-12T09:30:38,915 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2023-11-12T09:30:38,915 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_table1 Metrics about Tables on a single HBase RegionServer 2023-11-12T09:30:38,944 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:38,946 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] handler.UnassignRegionHandler(119): Close 5bdf91d1fb4f5b624b93a3ddc48685df 2023-11-12T09:30:38,946 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1668): Closing 5bdf91d1fb4f5b624b93a3ddc48685df, disabling compactions & flushes 2023-11-12T09:30:38,946 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1690): Closing region table1,,1699781433694.5bdf91d1fb4f5b624b93a3ddc48685df. 2023-11-12T09:30:38,946 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1699781433694.5bdf91d1fb4f5b624b93a3ddc48685df. 2023-11-12T09:30:38,946 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1778): Acquired close lock on table1,,1699781433694.5bdf91d1fb4f5b624b93a3ddc48685df. after waiting 0 ms 2023-11-12T09:30:38,946 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1788): Updates disabled for region table1,,1699781433694.5bdf91d1fb4f5b624b93a3ddc48685df. 2023-11-12T09:30:38,955 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/table1/5bdf91d1fb4f5b624b93a3ddc48685df/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-11-12T09:30:38,956 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:30:38,957 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1909): Closed table1,,1699781433694.5bdf91d1fb4f5b624b93a3ddc48685df. 2023-11-12T09:30:38,957 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1622): Region close journal for 5bdf91d1fb4f5b624b93a3ddc48685df: 2023-11-12T09:30:38,959 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] handler.UnassignRegionHandler(163): Closed 5bdf91d1fb4f5b624b93a3ddc48685df 2023-11-12T09:30:38,959 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=5bdf91d1fb4f5b624b93a3ddc48685df, regionState=CLOSED 2023-11-12T09:30:38,963 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=41, resume processing ppid=40 2023-11-12T09:30:38,964 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=41, ppid=40, state=SUCCESS; CloseRegionProcedure 5bdf91d1fb4f5b624b93a3ddc48685df, server=jenkins-hbase3.apache.org,37573,1699781400943 in 178 msec 2023-11-12T09:30:38,965 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=40, resume processing ppid=39 2023-11-12T09:30:38,965 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=40, ppid=39, state=SUCCESS; TransitRegionStateProcedure table=table1, region=5bdf91d1fb4f5b624b93a3ddc48685df, UNASSIGN in 187 msec 2023-11-12T09:30:38,966 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781438966"}]},"ts":"1699781438966"} 2023-11-12T09:30:38,967 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLED in hbase:meta 2023-11-12T09:30:38,971 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(305): Set table1 to state=DISABLED 2023-11-12T09:30:38,973 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=39, state=SUCCESS; DisableTableProcedure table=table1 in 221 msec 2023-11-12T09:30:39,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=39 2023-11-12T09:30:39,071 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] client.HBaseAdmin$TableFuture(3745): Operation: DISABLE, Table Name: default:table1, procId: 39 completed 2023-11-12T09:30:39,076 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.HMaster$5(2449): Client=jenkins//172.31.12.81 delete table1 2023-11-12T09:30:39,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] procedure2.ProcedureExecutor(1032): Stored pid=42, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=table1 2023-11-12T09:30:39,091 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=42, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-11-12T09:30:39,093 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=42, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=table1 2023-11-12T09:30:39,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=42 2023-11-12T09:30:39,099 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/table1/5bdf91d1fb4f5b624b93a3ddc48685df 2023-11-12T09:30:39,104 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/table1/5bdf91d1fb4f5b624b93a3ddc48685df/f, FileablePath, hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/table1/5bdf91d1fb4f5b624b93a3ddc48685df/recovered.edits] 2023-11-12T09:30:39,111 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/table1/5bdf91d1fb4f5b624b93a3ddc48685df/f/d9277234f34f4479bea6840d3ef9d267_SeqId_4_ to hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/archive/data/default/table1/5bdf91d1fb4f5b624b93a3ddc48685df/f/d9277234f34f4479bea6840d3ef9d267_SeqId_4_ 2023-11-12T09:30:39,115 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/table1/5bdf91d1fb4f5b624b93a3ddc48685df/recovered.edits/6.seqid to hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/archive/data/default/table1/5bdf91d1fb4f5b624b93a3ddc48685df/recovered.edits/6.seqid 2023-11-12T09:30:39,115 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/table1/5bdf91d1fb4f5b624b93a3ddc48685df 2023-11-12T09:30:39,115 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(293): Archived table1 regions 2023-11-12T09:30:39,118 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=42, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=table1 2023-11-12T09:30:39,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37573 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2023-11-12T09:30:39,130 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of table1 from hbase:meta 2023-11-12T09:30:39,133 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(387): Removing 'table1' descriptor. 2023-11-12T09:30:39,134 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=42, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=table1 2023-11-12T09:30:39,134 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(377): Removing 'table1' from region states. 2023-11-12T09:30:39,135 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1,,1699781433694.5bdf91d1fb4f5b624b93a3ddc48685df.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1699781439134"}]},"ts":"9223372036854775807"} 2023-11-12T09:30:39,137 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-12T09:30:39,137 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 5bdf91d1fb4f5b624b93a3ddc48685df, NAME => 'table1,,1699781433694.5bdf91d1fb4f5b624b93a3ddc48685df.', STARTKEY => '', ENDKEY => ''}] 2023-11-12T09:30:39,137 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(381): Marking 'table1' as deleted. 2023-11-12T09:30:39,138 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1699781439137"}]},"ts":"9223372036854775807"} 2023-11-12T09:30:39,140 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1678): Deleted table table1 state from META 2023-11-12T09:30:39,143 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(127): Finished pid=42, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-11-12T09:30:39,145 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=42, state=SUCCESS; DeleteTableProcedure table=table1 in 64 msec 2023-11-12T09:30:39,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=42 2023-11-12T09:30:39,196 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] client.HBaseAdmin$TableFuture(3745): Operation: DELETE, Table Name: default:table1, procId: 42 completed 2023-11-12T09:30:39,221 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.ResourceChecker(175): after: backup.TestRemoteRestore#testFullRestoreRemote Thread=831 (was 814) Potentially hanging thread: hconnection-0x2c8ba79b-shared-pool-11 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1479845225_16 at /127.0.0.1:35604 [Waiting for operation #8] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x2c8ba79b-shared-pool-9 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer for 'JobHistoryServer' metrics system java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: (jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS-EventLoopGroup-5-3 org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:209) org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:202) org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:316) org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:373) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Client (405311212) connection to localhost/127.0.0.1:41663 from jenkins.hfs.1 java.lang.Object.wait(Native Method) org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1053) org.apache.hadoop.ipc.Client$Connection.run(Client.java:1097) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-278318368_16 at /127.0.0.1:47698 [Receiving block BP-416291311-172.31.12.81-1699781394581:blk_1073741856_1032] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x2c8ba79b-shared-pool-6 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-416291311-172.31.12.81-1699781394581:blk_1073741855_1031, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: member: 'jenkins-hbase3.apache.org,37573,1699781400943' subprocedure-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/cluster_1b67588a-7841-a8ad-b22d-cb19219656d1/dfs/data/data1/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x2c8ba79b-shared-pool-7 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/cluster_1b67588a-7841-a8ad-b22d-cb19219656d1/dfs/data/data2/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x2c8ba79b-shared-pool-8 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,37573,1699781400943)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: HFileArchiver-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-278318368_16 at /127.0.0.1:47686 [Receiving block BP-416291311-172.31.12.81-1699781394581:blk_1073741855_1031] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_433146834_16 at /127.0.0.1:60122 [Waiting for operation #4] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x2c8ba79b-shared-pool-10 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-416291311-172.31.12.81-1699781394581:blk_1073741856_1032, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=935 (was 907) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=183 (was 163) - SystemLoadAverage LEAK? -, ProcessCount=169 (was 169), AvailableMemoryMB=5366 (was 5587) 2023-11-12T09:30:39,221 WARN [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.ResourceChecker(130): Thread=831 is superior to 500 2023-11-12T09:30:39,243 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.ResourceChecker(147): before: backup.TestRemoteRestore#testFullRestoreRemoteWithAlternateRestoreOutputDir Thread=831, OpenFileDescriptor=935, MaxFileDescriptor=60000, SystemLoadAverage=183, ProcessCount=169, AvailableMemoryMB=5365 2023-11-12T09:30:39,243 WARN [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.ResourceChecker(130): Thread=831 is superior to 500 2023-11-12T09:30:39,243 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] backup.TestRemoteRestore(91): test remote full backup on a single table with alternate restore output dir 2023-11-12T09:30:39,244 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0087f5b1 to 127.0.0.1:57951 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:30:39,252 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cde44c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:30:39,255 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:30:39,257 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:60336, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:30:39,259 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:30:39,261 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:46554, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:30:39,275 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:30:39,281 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:30:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781439281 (current time:1699781439281). 2023-11-12T09:30:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:30:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:30:39,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x75234037 to 127.0.0.1:57951 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:30:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4021b99c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:30:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:30:39,294 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:60342, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:30:39,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x75234037 to 127.0.0.1:57951 2023-11-12T09:30:39,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:30:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:30:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] procedure2.ProcedureExecutor(1032): Stored pid=43, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:30:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 43 2023-11-12T09:30:39,299 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:30:39,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=43 2023-11-12T09:30:39,300 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:30:39,303 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/.hbase-snapshot, current working dir path: hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor176.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor176.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:30:39,305 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:30:39,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=43 2023-11-12T09:30:39,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=43 2023-11-12T09:30:39,712 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:30:39,712 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE; SnapshotRegionProcedure 441fc40990b285ba0c11d5ae2272b2af}] 2023-11-12T09:30:39,715 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=44, ppid=43, state=RUNNABLE; SnapshotRegionProcedure 441fc40990b285ba0c11d5ae2272b2af 2023-11-12T09:30:39,865 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:39,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37573 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=44 2023-11-12T09:30:39,872 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1699781421319.441fc40990b285ba0c11d5ae2272b2af. 2023-11-12T09:30:39,873 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegion(2819): Flushing 441fc40990b285ba0c11d5ae2272b2af 2/2 column families, dataSize=1.08 KB heapSize=2.23 KB 2023-11-12T09:30:39,894 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/.tmp/meta/001a1045334043148e41ffc58e2cd8e7 is 169, key is trslm:hdfs://localhost:41663/backupUT\x00test-1699781414320/meta:log-roll-map/1699781433160/Put/seqid=0 2023-11-12T09:30:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=43 2023-11-12T09:30:40,316 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=526 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/.tmp/meta/001a1045334043148e41ffc58e2cd8e7 2023-11-12T09:30:40,325 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/.tmp/session/5fdd4ef141bb42069437ecf8e9c6f367 is 310, key is session:backup_1699781421145/session:context/1699781433633/Put/seqid=0 2023-11-12T09:30:40,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=43 2023-11-12T09:30:40,457 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:30:40,731 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=579 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/.tmp/session/5fdd4ef141bb42069437ecf8e9c6f367 2023-11-12T09:30:40,738 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/.tmp/meta/001a1045334043148e41ffc58e2cd8e7 as hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/meta/001a1045334043148e41ffc58e2cd8e7 2023-11-12T09:30:40,745 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/meta/001a1045334043148e41ffc58e2cd8e7, entries=4, sequenceid=17, filesize=5.5 K 2023-11-12T09:30:40,746 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/.tmp/session/5fdd4ef141bb42069437ecf8e9c6f367 as hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/session/5fdd4ef141bb42069437ecf8e9c6f367 2023-11-12T09:30:40,751 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/session/5fdd4ef141bb42069437ecf8e9c6f367, entries=2, sequenceid=17, filesize=5.3 K 2023-11-12T09:30:40,754 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegion(3022): Finished flush of dataSize ~1.08 KB/1105, heapSize ~2.20 KB/2256, currentSize=0 B/0 for 441fc40990b285ba0c11d5ae2272b2af in 881ms, sequenceid=17, compaction requested=false 2023-11-12T09:30:40,754 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegion(2520): Flush status journal for 441fc40990b285ba0c11d5ae2272b2af: 2023-11-12T09:30:40,754 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1699781421319.441fc40990b285ba0c11d5ae2272b2af. for snapshot_backup_system completed. 2023-11-12T09:30:40,754 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1699781421319.441fc40990b285ba0c11d5ae2272b2af.' region-info for snapshot=snapshot_backup_system 2023-11-12T09:30:40,754 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:30:40,754 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/meta/001a1045334043148e41ffc58e2cd8e7] hfiles 2023-11-12T09:30:40,754 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/meta/001a1045334043148e41ffc58e2cd8e7 for snapshot=snapshot_backup_system 2023-11-12T09:30:40,754 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/session/69aeda0ec11340f6be24d8bb52e8f078, hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/session/5fdd4ef141bb42069437ecf8e9c6f367] hfiles 2023-11-12T09:30:40,755 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/session/69aeda0ec11340f6be24d8bb52e8f078 for snapshot=snapshot_backup_system 2023-11-12T09:30:40,755 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/session/5fdd4ef141bb42069437ecf8e9c6f367 for snapshot=snapshot_backup_system 2023-11-12T09:30:41,160 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1699781421319.441fc40990b285ba0c11d5ae2272b2af. 2023-11-12T09:30:41,160 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=44 2023-11-12T09:30:41,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.HMaster(4026): Remote procedure done, pid=44 2023-11-12T09:30:41,161 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 441fc40990b285ba0c11d5ae2272b2af 2023-11-12T09:30:41,163 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=44, ppid=43, state=RUNNABLE; SnapshotRegionProcedure 441fc40990b285ba0c11d5ae2272b2af 2023-11-12T09:30:41,166 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=44, resume processing ppid=43 2023-11-12T09:30:41,166 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:30:41,166 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=44, ppid=43, state=SUCCESS; SnapshotRegionProcedure 441fc40990b285ba0c11d5ae2272b2af in 1.4520 sec 2023-11-12T09:30:41,166 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:30:41,167 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:30:41,167 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-12T09:30:41,168 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-12T09:30:41,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=43 2023-11-12T09:30:41,578 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:30:41,586 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:30:41,586 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/.hbase-snapshot/snapshot_backup_system 2023-11-12T09:30:41,587 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:30:41,587 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 43 2023-11-12T09:30:41,588 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=43, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2910 sec 2023-11-12T09:30:43,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=43 2023-11-12T09:30:43,407 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: backup:system, procId: 43 completed 2023-11-12T09:30:43,407 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] impl.TableBackupClient(115): Backup backup_1699781439252 started at 1699781443407. 2023-11-12T09:30:43,409 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.TableBackupClient(119): Backup session backup_1699781439252 has been started. 2023-11-12T09:30:43,414 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-12T09:30:43,415 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(962): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-11-12T09:30:43,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-12T09:30:43,415 INFO [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-12T09:30:43,415 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-12T09:30:43,417 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-12T09:30:43,417 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-12T09:30:43,417 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-12T09:30:43,417 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:30:43,419 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-12T09:30:43,419 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:43,419 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:30:43,419 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:30:43,419 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:43,419 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-12T09:30:43,419 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-12T09:30:43,420 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-12T09:30:43,420 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-12T09:30:43,420 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:30:43,420 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-12T09:30:43,420 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-12T09:30:43,420 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-12T09:30:43,420 DEBUG [member: 'jenkins-hbase3.apache.org,37573,1699781400943' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-12T09:30:43,421 DEBUG [member: 'jenkins-hbase3.apache.org,37573,1699781400943' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-12T09:30:43,421 DEBUG [member: 'jenkins-hbase3.apache.org,37573,1699781400943' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-12T09:30:43,421 DEBUG [member: 'jenkins-hbase3.apache.org,37573,1699781400943' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-12T09:30:43,421 DEBUG [member: 'jenkins-hbase3.apache.org,37573,1699781400943' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,37573,1699781400943' joining acquired barrier for procedure (rolllog) in zk 2023-11-12T09:30:43,423 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:43,423 DEBUG [member: 'jenkins-hbase3.apache.org,37573,1699781400943' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-12T09:30:43,423 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:43,423 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:30:43,423 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:30:43,424 DEBUG [member: 'jenkins-hbase3.apache.org,37573,1699781400943' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-11-12T09:30:43,424 DEBUG [member: 'jenkins-hbase3.apache.org,37573,1699781400943' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-12T09:30:43,424 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:30:43,424 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:30:43,425 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:30:43,425 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:43,425 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:30:43,426 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,37573,1699781400943' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-12T09:30:43,426 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@4686aa9[Count = 0] remaining members to acquire global barrier 2023-11-12T09:30:43,426 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-12T09:30:43,426 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-12T09:30:43,427 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:30:43,427 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-12T09:30:43,427 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-12T09:30:43,427 DEBUG [member: 'jenkins-hbase3.apache.org,37573,1699781400943' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-12T09:30:43,428 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:43,428 DEBUG [member: 'jenkins-hbase3.apache.org,37573,1699781400943' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-12T09:30:43,428 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-12T09:30:43,428 DEBUG [rs(jenkins-hbase3.apache.org,37573,1699781400943)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:43,428 INFO [rs(jenkins-hbase3.apache.org,37573,1699781400943)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1699781427849 highest: 1699781427849 on jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:43,428 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37573%2C1699781400943.meta:.meta(num 1699781427824) roll requested 2023-11-12T09:30:43,439 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/WALs/jenkins-hbase3.apache.org,37573,1699781400943/jenkins-hbase3.apache.org%2C37573%2C1699781400943.meta.1699781443428.meta, exclude list is [], retry=0 2023-11-12T09:30:43,443 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44263,DS-4bdce35f-16ea-409e-9d44-e8db6ed957eb,DISK] 2023-11-12T09:30:43,445 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/WALs/jenkins-hbase3.apache.org,37573,1699781400943/jenkins-hbase3.apache.org%2C37573%2C1699781400943.meta.1699781427824.meta with entries=12, filesize=2.73 KB; new WAL /user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/WALs/jenkins-hbase3.apache.org,37573,1699781400943/jenkins-hbase3.apache.org%2C37573%2C1699781400943.meta.1699781443428.meta 2023-11-12T09:30:43,445 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:44263,DS-4bdce35f-16ea-409e-9d44-e8db6ed957eb,DISK]] 2023-11-12T09:30:43,445 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/WALs/jenkins-hbase3.apache.org,37573,1699781400943/jenkins-hbase3.apache.org%2C37573%2C1699781400943.meta.1699781427824.meta is not closed yet, will try archiving it next time 2023-11-12T09:30:43,445 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37573%2C1699781400943:(num 1699781427849) roll requested 2023-11-12T09:30:43,448 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/WALs/jenkins-hbase3.apache.org,37573,1699781400943/jenkins-hbase3.apache.org%2C37573%2C1699781400943.meta.1699781427824.meta not finished, retry = 0 2023-11-12T09:30:43,458 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/WALs/jenkins-hbase3.apache.org,37573,1699781400943/jenkins-hbase3.apache.org%2C37573%2C1699781400943.1699781443446, exclude list is [], retry=0 2023-11-12T09:30:43,464 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44263,DS-4bdce35f-16ea-409e-9d44-e8db6ed957eb,DISK] 2023-11-12T09:30:43,466 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/WALs/jenkins-hbase3.apache.org,37573,1699781400943/jenkins-hbase3.apache.org%2C37573%2C1699781400943.1699781427849 with entries=15, filesize=3.47 KB; new WAL /user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/WALs/jenkins-hbase3.apache.org,37573,1699781400943/jenkins-hbase3.apache.org%2C37573%2C1699781400943.1699781443446 2023-11-12T09:30:43,466 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:44263,DS-4bdce35f-16ea-409e-9d44-e8db6ed957eb,DISK]] 2023-11-12T09:30:43,466 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/WALs/jenkins-hbase3.apache.org,37573,1699781400943/jenkins-hbase3.apache.org%2C37573%2C1699781400943.1699781427849 is not closed yet, will try archiving it next time 2023-11-12T09:30:43,468 DEBUG [rs(jenkins-hbase3.apache.org,37573,1699781400943)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-11-12T09:30:43,468 INFO [rs(jenkins-hbase3.apache.org,37573,1699781400943)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1699781443446 on jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:43,478 DEBUG [member: 'jenkins-hbase3.apache.org,37573,1699781400943' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-12T09:30:43,478 DEBUG [member: 'jenkins-hbase3.apache.org,37573,1699781400943' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,37573,1699781400943' in zk 2023-11-12T09:30:43,481 DEBUG [member: 'jenkins-hbase3.apache.org,37573,1699781400943' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-12T09:30:43,481 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:43,481 DEBUG [member: 'jenkins-hbase3.apache.org,37573,1699781400943' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:30:43,481 DEBUG [member: 'jenkins-hbase3.apache.org,37573,1699781400943' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-12T09:30:43,482 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:43,482 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:30:43,482 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:30:43,483 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:30:43,483 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:30:43,483 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:30:43,484 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:43,484 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:30:43,484 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:30:43,485 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:43,485 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,37573,1699781400943': 2023-11-12T09:30:43,485 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,37573,1699781400943' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-12T09:30:43,485 INFO [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-12T09:30:43,485 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-12T09:30:43,485 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-12T09:30:43,485 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-12T09:30:43,485 INFO [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-12T09:30:43,487 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:30:43,487 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:30:43,487 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-12T09:30:43,487 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-12T09:30:43,488 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:30:43,488 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-12T09:30:43,488 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:30:43,488 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:30:43,488 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:30:43,488 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:30:43,488 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:30:43,488 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:43,488 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:30:43,488 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:30:43,489 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:30:43,489 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:30:43,490 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:43,490 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:43,490 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:30:43,490 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:30:43,491 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:43,493 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:43,493 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-12T09:30:43,493 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-12T09:30:43,493 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-12T09:30:43,493 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-12T09:30:43,493 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:43,493 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:30:43,493 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:30:43,493 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:30:43,493 DEBUG [(jenkins-hbase3.apache.org,44289,1699781397811)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:30:43,493 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-12T09:30:43,493 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:30:43,493 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-12T09:30:43,494 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:30:43,494 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] client.HBaseAdmin(2906): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-11-12T09:30:43,494 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:30:43,495 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] client.HBaseAdmin(2914): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-11-12T09:30:43,495 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:30:43,595 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] client.HBaseAdmin(2920): Getting current status of procedure from master... 2023-11-12T09:30:43,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-12T09:30:43,599 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1699781443598_default_test-1699781414320 table=test-1699781414320 type=FLUSH ttl=0 } 2023-11-12T09:30:43,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781443599 (current time:1699781443599). 2023-11-12T09:30:43,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:30:43,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1699781443598_default_test-1699781414320 VERSION not specified, setting to 2 2023-11-12T09:30:43,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x12e95579 to 127.0.0.1:57951 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:30:43,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c503194, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:30:43,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:30:43,616 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:60358, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:30:43,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x12e95579 to 127.0.0.1:57951 2023-11-12T09:30:43,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:30:43,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:30:43,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] procedure2.ProcedureExecutor(1032): Stored pid=45, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1699781443598_default_test-1699781414320 table=test-1699781414320 type=FLUSH ttl=0 } 2023-11-12T09:30:43,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1699781443598_default_test-1699781414320 table=test-1699781414320 type=FLUSH ttl=0 }, snapshot procedure id = 45 2023-11-12T09:30:43,625 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1699781443598_default_test-1699781414320 table=test-1699781414320 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:30:43,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=45 2023-11-12T09:30:43,626 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1699781443598_default_test-1699781414320 table=test-1699781414320 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:30:43,629 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/.hbase-snapshot, current working dir path: hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/.hbase-snapshot/.tmp/snapshot_1699781443598_default_test-1699781414320 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor176.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor176.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor176.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor176.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor176.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor176.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:30:43,631 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1699781443598_default_test-1699781414320 table=test-1699781414320 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:30:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=45 2023-11-12T09:30:43,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=45 2023-11-12T09:30:44,038 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1699781443598_default_test-1699781414320 table=test-1699781414320 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:30:44,038 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE; SnapshotRegionProcedure 37c7f0bb858c8098fd397daa8ed9cc98}] 2023-11-12T09:30:44,040 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=46, ppid=45, state=RUNNABLE; SnapshotRegionProcedure 37c7f0bb858c8098fd397daa8ed9cc98 2023-11-12T09:30:44,191 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:44,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37573 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=46 2023-11-12T09:30:44,192 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1699781414320,,1699781414853.37c7f0bb858c8098fd397daa8ed9cc98. 2023-11-12T09:30:44,193 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] regionserver.HRegion(2520): Flush status journal for 37c7f0bb858c8098fd397daa8ed9cc98: 2023-11-12T09:30:44,193 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1699781414320,,1699781414853.37c7f0bb858c8098fd397daa8ed9cc98. for snapshot_1699781443598_default_test-1699781414320 completed. 2023-11-12T09:30:44,193 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] snapshot.SnapshotManifest(240): Storing 'test-1699781414320,,1699781414853.37c7f0bb858c8098fd397daa8ed9cc98.' region-info for snapshot=snapshot_1699781443598_default_test-1699781414320 2023-11-12T09:30:44,193 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:30:44,193 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/test-1699781414320/37c7f0bb858c8098fd397daa8ed9cc98/f/0502255d783947ce89e78b17fc613b51] hfiles 2023-11-12T09:30:44,193 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/test-1699781414320/37c7f0bb858c8098fd397daa8ed9cc98/f/0502255d783947ce89e78b17fc613b51 for snapshot=snapshot_1699781443598_default_test-1699781414320 2023-11-12T09:30:44,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=45 2023-11-12T09:30:44,599 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1699781414320,,1699781414853.37c7f0bb858c8098fd397daa8ed9cc98. 2023-11-12T09:30:44,599 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=46 2023-11-12T09:30:44,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.HMaster(4026): Remote procedure done, pid=46 2023-11-12T09:30:44,599 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1699781443598_default_test-1699781414320 on region 37c7f0bb858c8098fd397daa8ed9cc98 2023-11-12T09:30:44,601 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=46, ppid=45, state=RUNNABLE; SnapshotRegionProcedure 37c7f0bb858c8098fd397daa8ed9cc98 2023-11-12T09:30:44,603 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=46, resume processing ppid=45 2023-11-12T09:30:44,603 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=46, ppid=45, state=SUCCESS; SnapshotRegionProcedure 37c7f0bb858c8098fd397daa8ed9cc98 in 564 msec 2023-11-12T09:30:44,603 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1699781443598_default_test-1699781414320 table=test-1699781414320 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:30:44,604 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1699781443598_default_test-1699781414320 table=test-1699781414320 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:30:44,605 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1699781443598_default_test-1699781414320 table=test-1699781414320 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:30:44,605 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1699781443598_default_test-1699781414320 2023-11-12T09:30:44,606 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/.hbase-snapshot/.tmp/snapshot_1699781443598_default_test-1699781414320 2023-11-12T09:30:44,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=45 2023-11-12T09:30:45,015 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1699781443598_default_test-1699781414320 table=test-1699781414320 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:30:45,022 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1699781443598_default_test-1699781414320 table=test-1699781414320 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:30:45,022 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/.hbase-snapshot/.tmp/snapshot_1699781443598_default_test-1699781414320 to hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/.hbase-snapshot/snapshot_1699781443598_default_test-1699781414320 2023-11-12T09:30:45,024 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1699781443598_default_test-1699781414320 table=test-1699781414320 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:30:45,024 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1699781443598_default_test-1699781414320 table=test-1699781414320 type=FLUSH ttl=0 }, snapshot procedure id = 45 2023-11-12T09:30:45,025 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=45, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1699781443598_default_test-1699781414320 table=test-1699781414320 type=FLUSH ttl=0 } in 1.4060 sec 2023-11-12T09:30:45,271 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-11-12T09:30:45,271 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-11-12T09:30:45,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=45 2023-11-12T09:30:45,732 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: default:test-1699781414320, procId: 45 completed 2023-11-12T09:30:45,732 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1699781439252 2023-11-12T09:30:45,732 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-12T09:30:45,732 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-11-12T09:30:45,732 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1699781439252_test-1699781414320 2023-11-12T09:30:45,732 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1699781443598_default_test-1699781414320 to hdfs://localhost:41663/backupUT/backup_1699781439252/default/test-1699781414320/ 2023-11-12T09:30:45,732 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-12T09:30:45,756 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:38625, inputRoot=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a 2023-11-12T09:30:45,756 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,44289,1699781397811_-1138988997_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:41663/backupUT/backup_1699781439252/default/test-1699781414320, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:41663/backupUT/backup_1699781439252/default/test-1699781414320/.hbase-snapshot/.tmp/snapshot_1699781443598_default_test-1699781414320 2023-11-12T09:30:45,756 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:38625, inputRoot=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a, snapshotDir=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/.hbase-snapshot/snapshot_1699781443598_default_test-1699781414320. 2023-11-12T09:30:45,768 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/.hbase-snapshot/snapshot_1699781443598_default_test-1699781414320 to hdfs://localhost:41663/backupUT/backup_1699781439252/default/test-1699781414320/.hbase-snapshot/.tmp/snapshot_1699781443598_default_test-1699781414320 2023-11-12T09:30:46,194 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:46,195 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:46,195 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:46,196 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:47,166 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/hadoop-6292536065190634899.jar 2023-11-12T09:30:47,166 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:47,167 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:47,167 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:47,168 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:47,168 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:47,168 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:47,169 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:47,169 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:47,170 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:47,170 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:30:47,171 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:30:47,171 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:30:47,172 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:30:47,172 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:30:47,173 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:30:47,173 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:30:47,173 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:30:47,174 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:30:47,174 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:30:47,175 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:30:47,175 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:30:47,176 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:30:47,176 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:30:47,177 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:30:47,177 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:30:47,178 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:30:47,178 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:30:47,179 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:30:47,207 WARN [Listener at jenkins-hbase3.apache.org/40129 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-12T09:30:47,227 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1699781443598_default_test-1699781414320' hfile list 2023-11-12T09:30:47,230 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-11-12T09:30:47,323 WARN [Listener at jenkins-hbase3.apache.org/40129 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/mapreduce.cluster.local.dir/1699781447278/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:30:47,324 WARN [Listener at jenkins-hbase3.apache.org/40129 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/mapreduce.cluster.local.dir/1699781447278/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:30:47,378 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-12T09:30:47,404 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/test-1699781414320/37c7f0bb858c8098fd397daa8ed9cc98/f/0502255d783947ce89e78b17fc613b51 output=hdfs://localhost:41663/backupUT/backup_1699781439252/default/test-1699781414320/archive/data/default/test-1699781414320/37c7f0bb858c8098fd397daa8ed9cc98/f/0502255d783947ce89e78b17fc613b51 2023-11-12T09:30:47,404 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 4.004M/sec 2023-11-12T09:30:48,360 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-12T09:30:48,361 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-12T09:30:48,369 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1699781443598_default_test-1699781414320 2023-11-12T09:30:48,369 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1699781443598_default_test-1699781414320 finished. 2023-11-12T09:30:48,369 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.BackupSystemTable(1034): test-1699781414320 2023-11-12T09:30:48,377 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781439252 2023-11-12T09:30:48,377 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-12T09:30:48,783 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:41663/backupUT/backup_1699781439252/.backup.manifest 2023-11-12T09:30:48,783 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.TableBackupClient(358): Backup backup_1699781439252 finished: type=FULL,tablelist=test-1699781414320,targetRootDir=hdfs://localhost:41663/backupUT,startts=1699781443407,completets=1699781448377,bytescopied=0 2023-11-12T09:30:48,783 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-12T09:30:48,783 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1699781443598_default_test-1699781414320 2023-11-12T09:30:48,784 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_1699781443598_default_test-1699781414320" 2023-11-12T09:30:48,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1699781443598_default_test-1699781414320 2023-11-12T09:30:48,788 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1699781443598_default_test-1699781414320 for backup backup_1699781439252 succeeded. 2023-11-12T09:30:48,789 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:30:48,794 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:30:48,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-12T09:30:48,797 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-12T09:30:48,798 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:30:48,804 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] impl.TableBackupClient(376): Backup backup_1699781439252 completed. 2023-11-12T09:30:48,804 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-12T09:30:48,805 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0087f5b1 to 127.0.0.1:57951 2023-11-12T09:30:48,805 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:30:48,805 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] backup.TestRemoteRestore(94): backup complete 2023-11-12T09:30:48,820 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/hadoop.log.dir so I do NOT create it in target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb 2023-11-12T09:30:48,820 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/hadoop.tmp.dir so I do NOT create it in target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb 2023-11-12T09:30:48,820 WARN [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.HBaseTestingUtility(455): hadoop.tmp.dir property value differs in configuration and system: Configuration=/tmp/hadoop-jenkins while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/hadoop.tmp.dir Erasing configuration value by system value. 2023-11-12T09:30:48,820 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb 2023-11-12T09:30:48,820 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/cluster_029c07f3-9135-3e2b-fa26-4408331c19ee, deleteOnExit=true 2023-11-12T09:30:48,820 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/test.cache.data in system properties and HBase conf 2023-11-12T09:30:48,820 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/hadoop.tmp.dir in system properties and HBase conf 2023-11-12T09:30:48,820 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/hadoop.log.dir in system properties and HBase conf 2023-11-12T09:30:48,821 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/mapreduce.cluster.local.dir in system properties and HBase conf 2023-11-12T09:30:48,821 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-11-12T09:30:48,821 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-11-12T09:30:48,821 DEBUG [Listener at jenkins-hbase3.apache.org/40129 {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-11-12T09:30:48,821 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-11-12T09:30:48,821 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-11-12T09:30:48,821 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-11-12T09:30:48,822 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-12T09:30:48,822 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-11-12T09:30:48,822 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-11-12T09:30:48,822 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-12T09:30:48,822 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-12T09:30:48,822 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-11-12T09:30:48,822 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/nfs.dump.dir in system properties and HBase conf 2023-11-12T09:30:48,822 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/java.io.tmpdir in system properties and HBase conf 2023-11-12T09:30:48,822 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-12T09:30:48,822 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-11-12T09:30:48,822 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-11-12T09:30:48,834 WARN [Listener at jenkins-hbase3.apache.org/40129 {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-11-12T09:30:48,834 WARN [Listener at jenkins-hbase3.apache.org/40129 {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-11-12T09:30:48,885 WARN [Listener at jenkins-hbase3.apache.org/40129 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:30:48,887 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:30:48,894 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/java.io.tmpdir/Jetty_localhost_35117_hdfs____uylogv/webapp 2023-11-12T09:30:48,915 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2023-11-12T09:30:48,995 INFO [Listener at jenkins-hbase3.apache.org/40129 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:35117 2023-11-12T09:30:49,007 WARN [Listener at jenkins-hbase3.apache.org/40129 {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-11-12T09:30:49,008 WARN [Listener at jenkins-hbase3.apache.org/40129 {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-11-12T09:30:49,060 WARN [Listener at localhost/45497 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-11-12T09:30:49,075 WARN [Listener at localhost/45497 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-11-12T09:30:49,077 WARN [Listener at localhost/45497 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:30:49,078 INFO [Listener at localhost/45497 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:30:49,084 INFO [Listener at localhost/45497 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/java.io.tmpdir/Jetty_localhost_37977_datanode____.7h9w9w/webapp 2023-11-12T09:30:49,180 INFO [Listener at localhost/45497 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:37977 2023-11-12T09:30:49,196 WARN [Listener at localhost/39907 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-11-12T09:30:49,211 WARN [Listener at localhost/39907 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-11-12T09:30:49,212 WARN [Listener at localhost/39907 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:30:49,214 INFO [Listener at localhost/39907 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:30:49,220 INFO [Listener at localhost/39907 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/java.io.tmpdir/Jetty_localhost_40557_datanode____.902nlq/webapp 2023-11-12T09:30:49,266 WARN [Thread-1113 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-11-12T09:30:49,299 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xe67b501bb52a66d4: Processing first storage report for DS-fbb435c6-66b6-4163-b467-6fc926ef34e6 from datanode 01e68a4e-547b-44e0-ab3b-5ea1df68c40d 2023-11-12T09:30:49,299 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xe67b501bb52a66d4: from storage DS-fbb435c6-66b6-4163-b467-6fc926ef34e6 node DatanodeRegistration(127.0.0.1:45301, datanodeUuid=01e68a4e-547b-44e0-ab3b-5ea1df68c40d, infoPort=43375, infoSecurePort=0, ipcPort=39907, storageInfo=lv=-57;cid=testClusterID;nsid=439979335;c=1699781448837), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2023-11-12T09:30:49,299 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xe67b501bb52a66d4: Processing first storage report for DS-1795b506-c16c-4c49-b67b-849c63caaf13 from datanode 01e68a4e-547b-44e0-ab3b-5ea1df68c40d 2023-11-12T09:30:49,299 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xe67b501bb52a66d4: from storage DS-1795b506-c16c-4c49-b67b-849c63caaf13 node DatanodeRegistration(127.0.0.1:45301, datanodeUuid=01e68a4e-547b-44e0-ab3b-5ea1df68c40d, infoPort=43375, infoSecurePort=0, ipcPort=39907, storageInfo=lv=-57;cid=testClusterID;nsid=439979335;c=1699781448837), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-11-12T09:30:49,332 INFO [Listener at localhost/39907 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:40557 2023-11-12T09:30:49,391 WARN [Listener at localhost/41963 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-11-12T09:30:49,408 WARN [Listener at localhost/41963 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-11-12T09:30:49,411 WARN [Listener at localhost/41963 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:30:49,412 INFO [Listener at localhost/41963 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:30:49,422 INFO [Listener at localhost/41963 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/java.io.tmpdir/Jetty_localhost_43279_datanode____6i8nj0/webapp 2023-11-12T09:30:49,477 WARN [Thread-1148 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-11-12T09:30:49,506 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x60bc4c62f39ded9e: Processing first storage report for DS-4543ac02-0ac1-47ef-b3b6-28461c150b79 from datanode f1d60f81-aef2-41c3-967f-c94284929199 2023-11-12T09:30:49,506 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x60bc4c62f39ded9e: from storage DS-4543ac02-0ac1-47ef-b3b6-28461c150b79 node DatanodeRegistration(127.0.0.1:42633, datanodeUuid=f1d60f81-aef2-41c3-967f-c94284929199, infoPort=33505, infoSecurePort=0, ipcPort=41963, storageInfo=lv=-57;cid=testClusterID;nsid=439979335;c=1699781448837), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2023-11-12T09:30:49,507 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x60bc4c62f39ded9e: Processing first storage report for DS-a54abb6b-4060-4dab-beef-a4c46b69db07 from datanode f1d60f81-aef2-41c3-967f-c94284929199 2023-11-12T09:30:49,507 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x60bc4c62f39ded9e: from storage DS-a54abb6b-4060-4dab-beef-a4c46b69db07 node DatanodeRegistration(127.0.0.1:42633, datanodeUuid=f1d60f81-aef2-41c3-967f-c94284929199, infoPort=33505, infoSecurePort=0, ipcPort=41963, storageInfo=lv=-57;cid=testClusterID;nsid=439979335;c=1699781448837), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-11-12T09:30:49,530 INFO [Listener at localhost/41963 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:43279 2023-11-12T09:30:49,548 WARN [Listener at localhost/45101 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-11-12T09:30:49,637 WARN [Thread-1182 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-11-12T09:30:49,670 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x9c2fabc8d27c037a: Processing first storage report for DS-b334c032-90fb-4aec-b9ea-62f6d84d30a1 from datanode dbee65f5-8312-4a48-939d-4ae54d8ce622 2023-11-12T09:30:49,670 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x9c2fabc8d27c037a: from storage DS-b334c032-90fb-4aec-b9ea-62f6d84d30a1 node DatanodeRegistration(127.0.0.1:36257, datanodeUuid=dbee65f5-8312-4a48-939d-4ae54d8ce622, infoPort=34985, infoSecurePort=0, ipcPort=45101, storageInfo=lv=-57;cid=testClusterID;nsid=439979335;c=1699781448837), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2023-11-12T09:30:49,670 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x9c2fabc8d27c037a: Processing first storage report for DS-baa4f245-22a9-4651-9cb4-97b0f9c03f39 from datanode dbee65f5-8312-4a48-939d-4ae54d8ce622 2023-11-12T09:30:49,670 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x9c2fabc8d27c037a: from storage DS-baa4f245-22a9-4651-9cb4-97b0f9c03f39 node DatanodeRegistration(127.0.0.1:36257, datanodeUuid=dbee65f5-8312-4a48-939d-4ae54d8ce622, infoPort=34985, infoSecurePort=0, ipcPort=45101, storageInfo=lv=-57;cid=testClusterID;nsid=439979335;c=1699781448837), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-11-12T09:30:49,684 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-11-12T09:30:49,684 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-11-12T09:30:49,759 DEBUG [Listener at localhost/45101 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb 2023-11-12T09:30:49,759 INFO [Listener at localhost/45101 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-11-12T09:30:49,759 INFO [Listener at localhost/45101 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/test.cache.data in system properties and HBase conf 2023-11-12T09:30:49,759 INFO [Listener at localhost/45101 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/hadoop.tmp.dir in system properties and HBase conf 2023-11-12T09:30:49,759 INFO [Listener at localhost/45101 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/hadoop.log.dir in system properties and HBase conf 2023-11-12T09:30:49,759 INFO [Listener at localhost/45101 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/mapreduce.cluster.local.dir in system properties and HBase conf 2023-11-12T09:30:49,759 INFO [Listener at localhost/45101 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-11-12T09:30:49,759 INFO [Listener at localhost/45101 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-11-12T09:30:49,759 INFO [Listener at localhost/45101 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:30:49,761 INFO [Listener at localhost/45101 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:30:49,764 INFO [Listener at localhost/45101 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-11-12T09:30:49,764 INFO [Listener at localhost/45101 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-11-12T09:30:49,764 INFO [Listener at localhost/45101 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-11-12T09:30:49,764 INFO [Listener at localhost/45101 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-12T09:30:49,764 INFO [Listener at localhost/45101 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-11-12T09:30:49,765 INFO [Listener at localhost/45101 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-11-12T09:30:49,765 INFO [Listener at localhost/45101 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-12T09:30:49,765 INFO [Listener at localhost/45101 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-12T09:30:49,765 INFO [Listener at localhost/45101 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-11-12T09:30:49,765 INFO [Listener at localhost/45101 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/nfs.dump.dir in system properties and HBase conf 2023-11-12T09:30:49,765 INFO [Listener at localhost/45101 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/java.io.tmpdir in system properties and HBase conf 2023-11-12T09:30:49,765 INFO [Listener at localhost/45101 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-12T09:30:49,765 INFO [Listener at localhost/45101 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-11-12T09:30:49,765 INFO [Listener at localhost/45101 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-11-12T09:30:50,292 WARN [Thread-1301 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:30:50,296 WARN [Thread-1301 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-12T09:30:50,297 INFO [Thread-1301 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:30:50,303 INFO [Thread-1301 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_43013_jobhistory____wi7y4d/webapp 2023-11-12T09:30:50,352 INFO [Thread-1301 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-11-12T09:30:50,352 INFO [Thread-1301 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-11-12T09:30:50,352 INFO [Thread-1301 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-12T09:30:50,352 INFO [Thread-1301 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-12T09:30:50,359 INFO [Thread-1301 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:30:50,400 INFO [Thread-1301 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:30:50,454 INFO [Thread-1301 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-11-12T09:30:50,460 INFO [Thread-1301 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:43013 2023-11-12T09:30:51,781 WARN [Listener at jenkins-hbase3.apache.org/34091 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:30:51,787 WARN [Listener at jenkins-hbase3.apache.org/34091 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-12T09:30:51,788 INFO [Listener at jenkins-hbase3.apache.org/34091 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:30:51,796 INFO [Listener at jenkins-hbase3.apache.org/34091 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_41995_cluster____e4mi7f/webapp 2023-11-12T09:30:51,849 INFO [Listener at jenkins-hbase3.apache.org/34091 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-11-12T09:30:51,849 INFO [Listener at jenkins-hbase3.apache.org/34091 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-11-12T09:30:51,849 INFO [Listener at jenkins-hbase3.apache.org/34091 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-12T09:30:51,849 INFO [Listener at jenkins-hbase3.apache.org/34091 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-12T09:30:51,856 INFO [Listener at jenkins-hbase3.apache.org/34091 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:30:51,941 INFO [Listener at jenkins-hbase3.apache.org/34091 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:30:52,003 INFO [Listener at jenkins-hbase3.apache.org/34091 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:30:52,009 INFO [Listener at jenkins-hbase3.apache.org/34091 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:41995 2023-11-12T09:30:52,141 WARN [Listener at jenkins-hbase3.apache.org/46107 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:30:52,148 WARN [Listener at jenkins-hbase3.apache.org/46107 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-12T09:30:52,148 INFO [Listener at jenkins-hbase3.apache.org/46107 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:30:52,155 INFO [Listener at jenkins-hbase3.apache.org/46107 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_36367_node____u7wis0/webapp 2023-11-12T09:30:52,204 INFO [Listener at jenkins-hbase3.apache.org/46107 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-11-12T09:30:52,204 INFO [Listener at jenkins-hbase3.apache.org/46107 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-12T09:30:52,204 INFO [Listener at jenkins-hbase3.apache.org/46107 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-11-12T09:30:52,204 INFO [Listener at jenkins-hbase3.apache.org/46107 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-12T09:30:52,211 INFO [Listener at jenkins-hbase3.apache.org/46107 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:30:52,226 INFO [Listener at jenkins-hbase3.apache.org/46107 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:30:52,281 INFO [Listener at jenkins-hbase3.apache.org/46107 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:30:52,288 INFO [Listener at jenkins-hbase3.apache.org/46107 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:36367 2023-11-12T09:30:52,378 WARN [Listener at jenkins-hbase3.apache.org/39019 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:30:52,383 WARN [Listener at jenkins-hbase3.apache.org/39019 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-12T09:30:52,384 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:30:52,394 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_37751_node____p1vn0u/webapp 2023-11-12T09:30:52,446 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-11-12T09:30:52,446 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-12T09:30:52,446 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-11-12T09:30:52,446 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-12T09:30:52,454 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:30:52,470 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:30:52,556 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:30:52,565 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:37751 2023-11-12T09:30:52,572 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-11-12T09:30:52,573 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x56d1f692 to 127.0.0.1:57951 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:30:52,593 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c786b8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:30:52,596 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:30:52,601 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:52772, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:30:52,603 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] impl.RestoreTablesClient(101): HBase table table1 does not exist. It will be created during restore process 2023-11-12T09:30:52,604 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41663/backupUT/backup_1699781439252/.backup.manifest 2023-11-12T09:30:52,608 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781439252/.backup.manifest 2023-11-12T09:30:52,608 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41663/backupUT/backup_1699781439252/.backup.manifest 2023-11-12T09:30:52,610 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781439252/.backup.manifest 2023-11-12T09:30:52,610 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] impl.RestoreTablesClient(148): Restoring 'test-1699781414320' to 'table1' from full backup image hdfs://localhost:41663/backupUT/backup_1699781439252/default/test-1699781414320 2023-11-12T09:30:52,619 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] util.RestoreTool(487): Creating target table 'table1' 2023-11-12T09:30:52,619 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:41663/backupUT/backup_1699781439252/default/test-1699781414320/archive/data/default/test-1699781414320/37c7f0bb858c8098fd397daa8ed9cc98 2023-11-12T09:30:52,620 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:41663/backupUT/backup_1699781439252/default/test-1699781414320/archive/data/default/test-1699781414320/37c7f0bb858c8098fd397daa8ed9cc98/f in region [hdfs://localhost:41663/backupUT/backup_1699781439252/default/test-1699781414320/archive/data/default/test-1699781414320/37c7f0bb858c8098fd397daa8ed9cc98] 2023-11-12T09:30:52,624 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] util.RestoreTool(442): Trying to figure out region boundaries hfile=hdfs://localhost:41663/backupUT/backup_1699781439252/default/test-1699781414320/archive/data/default/test-1699781414320/37c7f0bb858c8098fd397daa8ed9cc98/f/0502255d783947ce89e78b17fc613b51 first=row0 last=row98 2023-11-12T09:30:52,625 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:30:52,629 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:47684, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:30:52,630 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:30:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] procedure2.ProcedureExecutor(1032): Stored pid=47, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=table1 2023-11-12T09:30:52,635 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=47, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:30:52,635 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 47 2023-11-12T09:30:52,636 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=47, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:30:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=47 2023-11-12T09:30:52,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=47 2023-11-12T09:30:52,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=47 2023-11-12T09:30:53,044 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 1c91405cdf5571de133464d086f5ac7f, NAME => 'table1,,1699781452630.1c91405cdf5571de133464d086f5ac7f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a 2023-11-12T09:30:53,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=47 2023-11-12T09:30:53,451 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(888): Instantiated table1,,1699781452630.1c91405cdf5571de133464d086f5ac7f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:30:53,451 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1668): Closing 1c91405cdf5571de133464d086f5ac7f, disabling compactions & flushes 2023-11-12T09:30:53,451 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1690): Closing region table1,,1699781452630.1c91405cdf5571de133464d086f5ac7f. 2023-11-12T09:30:53,451 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1699781452630.1c91405cdf5571de133464d086f5ac7f. 2023-11-12T09:30:53,451 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on table1,,1699781452630.1c91405cdf5571de133464d086f5ac7f. after waiting 0 ms 2023-11-12T09:30:53,451 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region table1,,1699781452630.1c91405cdf5571de133464d086f5ac7f. 2023-11-12T09:30:53,451 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1909): Closed table1,,1699781452630.1c91405cdf5571de133464d086f5ac7f. 2023-11-12T09:30:53,451 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1622): Region close journal for 1c91405cdf5571de133464d086f5ac7f: 2023-11-12T09:30:53,452 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=47, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:30:53,452 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"table1,,1699781452630.1c91405cdf5571de133464d086f5ac7f.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1699781453452"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781453452"}]},"ts":"1699781453452"} 2023-11-12T09:30:53,454 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:30:53,455 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=47, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:30:53,455 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781453455"}]},"ts":"1699781453455"} 2023-11-12T09:30:53,456 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLING in hbase:meta 2023-11-12T09:30:53,461 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=1c91405cdf5571de133464d086f5ac7f, ASSIGN}] 2023-11-12T09:30:53,464 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=1c91405cdf5571de133464d086f5ac7f, ASSIGN 2023-11-12T09:30:53,465 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=table1, region=1c91405cdf5571de133464d086f5ac7f, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37573,1699781400943; forceNewPlan=false, retain=false 2023-11-12T09:30:53,618 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=48 updating hbase:meta row=1c91405cdf5571de133464d086f5ac7f, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:53,620 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE; OpenRegionProcedure 1c91405cdf5571de133464d086f5ac7f, server=jenkins-hbase3.apache.org,37573,1699781400943}] 2023-11-12T09:30:53,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=47 2023-11-12T09:30:53,772 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:53,776 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(130): Open table1,,1699781452630.1c91405cdf5571de133464d086f5ac7f. 2023-11-12T09:30:53,776 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7254): Opening region: {ENCODED => 1c91405cdf5571de133464d086f5ac7f, NAME => 'table1,,1699781452630.1c91405cdf5571de133464d086f5ac7f.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:30:53,776 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:30:53,776 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 1c91405cdf5571de133464d086f5ac7f 2023-11-12T09:30:53,776 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(888): Instantiated table1,,1699781452630.1c91405cdf5571de133464d086f5ac7f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:30:53,776 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7296): checking encryption for 1c91405cdf5571de133464d086f5ac7f 2023-11-12T09:30:53,776 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7299): checking classloading for 1c91405cdf5571de133464d086f5ac7f 2023-11-12T09:30:53,778 INFO [StoreOpener-1c91405cdf5571de133464d086f5ac7f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 1c91405cdf5571de133464d086f5ac7f 2023-11-12T09:30:53,779 INFO [StoreOpener-1c91405cdf5571de133464d086f5ac7f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c91405cdf5571de133464d086f5ac7f columnFamilyName f 2023-11-12T09:30:53,779 DEBUG [StoreOpener-1c91405cdf5571de133464d086f5ac7f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:30:53,780 INFO [StoreOpener-1c91405cdf5571de133464d086f5ac7f-1 {}] regionserver.HStore(324): Store=1c91405cdf5571de133464d086f5ac7f/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:30:53,781 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/table1/1c91405cdf5571de133464d086f5ac7f 2023-11-12T09:30:53,781 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/table1/1c91405cdf5571de133464d086f5ac7f 2023-11-12T09:30:53,784 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1076): writing seq id for 1c91405cdf5571de133464d086f5ac7f 2023-11-12T09:30:53,786 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/table1/1c91405cdf5571de133464d086f5ac7f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:30:53,786 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): Opened 1c91405cdf5571de133464d086f5ac7f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10484241600, jitterRate=-0.023578912019729614}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:30:53,787 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(992): Region open journal for 1c91405cdf5571de133464d086f5ac7f: 2023-11-12T09:30:53,788 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2556): Post open deploy tasks for table1,,1699781452630.1c91405cdf5571de133464d086f5ac7f., pid=49, masterSystemTime=1699781453772 2023-11-12T09:30:53,789 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2583): Finished post open deploy task for table1,,1699781452630.1c91405cdf5571de133464d086f5ac7f. 2023-11-12T09:30:53,789 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(158): Opened table1,,1699781452630.1c91405cdf5571de133464d086f5ac7f. 2023-11-12T09:30:53,790 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=48 updating hbase:meta row=1c91405cdf5571de133464d086f5ac7f, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:30:53,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=49, resume processing ppid=48 2023-11-12T09:30:53,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=49, ppid=48, state=SUCCESS; OpenRegionProcedure 1c91405cdf5571de133464d086f5ac7f, server=jenkins-hbase3.apache.org,37573,1699781400943 in 171 msec 2023-11-12T09:30:53,794 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=48, resume processing ppid=47 2023-11-12T09:30:53,794 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=48, ppid=47, state=SUCCESS; TransitRegionStateProcedure table=table1, region=1c91405cdf5571de133464d086f5ac7f, ASSIGN in 331 msec 2023-11-12T09:30:53,794 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=47, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:30:53,794 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781453794"}]},"ts":"1699781453794"} 2023-11-12T09:30:53,796 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLED in hbase:meta 2023-11-12T09:30:53,798 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=47, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:30:53,799 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=47, state=SUCCESS; CreateTableProcedure table=table1 in 1.1680 sec 2023-11-12T09:30:54,679 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:30:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=47 2023-11-12T09:30:54,743 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: default:table1, procId: 47 completed 2023-11-12T09:30:54,766 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:41663/backupUT/backup_1699781439252/default/test-1699781414320/archive/data/default/test-1699781414320/37c7f0bb858c8098fd397daa8ed9cc98 from hbase tables test-1699781414320 to tables table1 2023-11-12T09:30:54,767 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.MapReduceRestoreJob(72): Restore test-1699781414320 into table1 2023-11-12T09:30:54,769 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:38625/backupUT/bulk_output-default-table1-1699781454767 from hdfs://localhost:41663/backupUT/backup_1699781439252/default/test-1699781414320/archive/data/default/test-1699781414320/37c7f0bb858c8098fd397daa8ed9cc98 2023-11-12T09:30:54,770 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1234d492 to 127.0.0.1:57951 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:30:54,861 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b9c95b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:30:54,862 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:30:54,888 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:47686, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:30:54,889 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-11-12T09:30:54,889 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table table1 2023-11-12T09:30:54,917 DEBUG [hconnection-0x731343d-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:30:54,921 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:52774, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:30:54,923 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for table1: 2023-11-12T09:30:54,923 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-11-12T09:30:54,923 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to /user/jenkins/hbase-staging/partitions_62fbeec1-45ee-4375-9940-e72deaff46f6 2023-11-12T09:30:55,056 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:55,057 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:55,057 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:55,058 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:56,032 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/hadoop-6051355694810975526.jar 2023-11-12T09:30:56,033 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:56,033 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:56,033 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:56,034 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:56,034 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:56,034 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:56,035 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:56,035 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:56,035 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:56,036 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:30:56,037 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:30:56,037 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:30:56,038 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:30:56,038 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:30:56,039 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:30:56,039 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:30:56,039 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:30:56,040 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:30:56,040 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:30:56,041 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:30:56,041 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:30:56,042 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:56,042 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:56,042 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:56,043 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:56,043 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:56,043 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:30:56,044 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:30:56,044 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: table1 2023-11-12T09:30:56,044 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-12T09:30:56,044 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1234d492 to 127.0.0.1:57951 2023-11-12T09:30:56,044 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:30:56,045 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-11-12T09:30:56,045 WARN [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-11-12T09:30:56,045 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:30:56,236 WARN [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-12T09:30:56,703 WARN [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-11-12T09:30:57,012 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'table1' 2023-11-12T09:30:57,064 WARN [SchedulerEventDispatcher:Event Processor {}] capacity.LeafQueue(815): maximum-am-resource-percent is insufficient to start a single application in queue, it is likely set too low. skipping enforcement to allow at least one application to start 2023-11-12T09:30:57,065 WARN [SchedulerEventDispatcher:Event Processor {}] capacity.LeafQueue(851): maximum-am-resource-percent is insufficient to start a single application in queue for user, it is likely set too low. skipping enforcement to allow at least one application to start 2023-11-12T09:30:57,465 INFO [Socket Reader #1 for port 0 {}] ipc.Server$Connection(2001): Auth successful for appattempt_1699781452016_0001_000001 (auth:SIMPLE) 2023-11-12T09:30:58,915 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2023-11-12T09:30:58,915 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_table1 Metrics about Tables on a single HBase RegionServer 2023-11-12T09:31:02,345 INFO [Socket Reader #1 for port 0 {}] ipc.Server$Connection(2001): Auth successful for appattempt_1699781452016_0001_000001 (auth:SIMPLE) 2023-11-12T09:31:03,592 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region daff3e7fcafd33ff24024100cbc1e9c9 changed from -1.0 to 0.0, refreshing cache 2023-11-12T09:31:04,621 INFO [Socket Reader #1 for port 0 {}] ipc.Server$Connection(2001): Auth successful for appattempt_1699781452016_0001_000001 (auth:SIMPLE) 2023-11-12T09:31:09,437 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2023-11-12T09:31:09,438 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 3e47234fadef92f1114692ec74b0ca02 changed from -1.0 to 0.0, refreshing cache 2023-11-12T09:31:10,508 INFO [Socket Reader #1 for port 0 {}] ipc.Server$Connection(2001): Auth successful for appattempt_1699781452016_0001_000001 (auth:SIMPLE) 2023-11-12T09:31:13,850 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:35884, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:31:14,267 INFO [Socket Reader #1 for port 0 {}] ipc.Server$Connection(2001): Auth successful for appattempt_1699781452016_0001_000001 (auth:SIMPLE) 2023-11-12T09:31:14,287 WARN [ContainersLauncher #1 {}] nodemanager.DefaultContainerExecutor(305): Exit code from container container_1699781452016_0001_01_000003 is : 143 2023-11-12T09:31:15,447 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:38625/backupUT/bulk_output-default-table1-1699781454767 2023-11-12T09:31:15,448 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4abfd431 to 127.0.0.1:57951 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:31:15,457 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7456cda2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:31:15,458 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:31:15,462 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:34032, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:31:15,477 WARN [Listener at jenkins-hbase3.apache.org/39019 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:38625/backupUT/bulk_output-default-table1-1699781454767/_SUCCESS 2023-11-12T09:31:15,488 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:31:15,495 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:35564, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:31:15,524 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:38625/backupUT/bulk_output-default-table1-1699781454767/f/6e1693bcb12342088f0aee7ca476540d first=Optional[row0] last=Optional[row98] 2023-11-12T09:31:15,526 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=table1,,1699781452630.1c91405cdf5571de133464d086f5ac7f., hostname=jenkins-hbase3.apache.org,37573,1699781400943, seqNum=2 for row with hfile group [{f,hdfs://localhost:38625/backupUT/bulk_output-default-table1-1699781454767/f/6e1693bcb12342088f0aee7ca476540d}] 2023-11-12T09:31:15,533 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:38625/backupUT/bulk_output-default-table1-1699781454767/f/6e1693bcb12342088f0aee7ca476540d for inclusion in 1c91405cdf5571de133464d086f5ac7f/f 2023-11-12T09:31:15,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-11-12T09:31:15,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] regionserver.HStore(625): Region bounds: first= last= 2023-11-12T09:31:15,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] regionserver.HRegion(2520): Flush status journal for 1c91405cdf5571de133464d086f5ac7f: 2023-11-12T09:31:15,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:38625/backupUT/bulk_output-default-table1-1699781454767/f/6e1693bcb12342088f0aee7ca476540d to hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/staging/jenkins__table1__jb42kvuajhbk2nhuqqmpn9ce70vq1the1b4dn2d7krcarjafkbsotuare6hl1h9m/f/6e1693bcb12342088f0aee7ca476540d 2023-11-12T09:31:15,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/staging/jenkins__table1__jb42kvuajhbk2nhuqqmpn9ce70vq1the1b4dn2d7krcarjafkbsotuare6hl1h9m/f/6e1693bcb12342088f0aee7ca476540d as hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/table1/1c91405cdf5571de133464d086f5ac7f/f/cfff3aa0cca7485eb1b6784ab68b50d7_SeqId_4_ 2023-11-12T09:31:15,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x683d17fd to 127.0.0.1:57951 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:31:15,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@729bc302, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:31:15,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:31:15,557 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:35574, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:31:15,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:31:15,562 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:34046, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:31:15,578 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-12T09:31:15,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x683d17fd to 127.0.0.1:57951 2023-11-12T09:31:15,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:31:15,579 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/staging/jenkins__table1__jb42kvuajhbk2nhuqqmpn9ce70vq1the1b4dn2d7krcarjafkbsotuare6hl1h9m/f/6e1693bcb12342088f0aee7ca476540d into 1c91405cdf5571de133464d086f5ac7f/f as hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/table1/1c91405cdf5571de133464d086f5ac7f/f/cfff3aa0cca7485eb1b6784ab68b50d7_SeqId_4_ - updating store file list. 2023-11-12T09:31:15,586 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/table1/1c91405cdf5571de133464d086f5ac7f/f/cfff3aa0cca7485eb1b6784ab68b50d7_SeqId_4_ into 1c91405cdf5571de133464d086f5ac7f/f 2023-11-12T09:31:15,587 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/staging/jenkins__table1__jb42kvuajhbk2nhuqqmpn9ce70vq1the1b4dn2d7krcarjafkbsotuare6hl1h9m/f/6e1693bcb12342088f0aee7ca476540d into 1c91405cdf5571de133464d086f5ac7f/f (new location: hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/table1/1c91405cdf5571de133464d086f5ac7f/f/cfff3aa0cca7485eb1b6784ab68b50d7_SeqId_4_) 2023-11-12T09:31:15,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/staging/jenkins__table1__jb42kvuajhbk2nhuqqmpn9ce70vq1the1b4dn2d7krcarjafkbsotuare6hl1h9m/f/6e1693bcb12342088f0aee7ca476540d 2023-11-12T09:31:15,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5e00c76a to 127.0.0.1:57951 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:31:15,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5115f107, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:31:15,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:31:15,599 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:35590, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-11-12T09:31:15,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:31:15,606 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:34060, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-11-12T09:31:15,625 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-12T09:31:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5e00c76a to 127.0.0.1:57951 2023-11-12T09:31:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37573 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:31:15,628 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-12T09:31:15,628 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x4abfd431 to 127.0.0.1:57951 2023-11-12T09:31:15,628 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:31:15,629 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-11-12T09:31:15,629 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-11-12T09:31:15,629 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] impl.RestoreTablesClient(233): Backup: backup_1699781439252 hdfs://localhost:41663/backupUT/backup_1699781439252/default/test-1699781414320/ 2023-11-12T09:31:15,629 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-11-12T09:31:15,636 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] client.HBaseAdmin$18(967): Started disable of table1 2023-11-12T09:31:15,636 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.HMaster$13(2695): Client=jenkins//172.31.12.81 disable table1 2023-11-12T09:31:15,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] procedure2.ProcedureExecutor(1032): Stored pid=50, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=table1 2023-11-12T09:31:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-12T09:31:15,648 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781475648"}]},"ts":"1699781475648"} 2023-11-12T09:31:15,650 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLING in hbase:meta 2023-11-12T09:31:15,652 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set table1 to state=DISABLING 2023-11-12T09:31:15,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=1c91405cdf5571de133464d086f5ac7f, UNASSIGN}] 2023-11-12T09:31:15,657 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=51, ppid=50, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=1c91405cdf5571de133464d086f5ac7f, UNASSIGN 2023-11-12T09:31:15,657 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=51 updating hbase:meta row=1c91405cdf5571de133464d086f5ac7f, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:31:15,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE; CloseRegionProcedure 1c91405cdf5571de133464d086f5ac7f, server=jenkins-hbase3.apache.org,37573,1699781400943}] 2023-11-12T09:31:15,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-12T09:31:15,811 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:31:15,812 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] handler.UnassignRegionHandler(119): Close 1c91405cdf5571de133464d086f5ac7f 2023-11-12T09:31:15,812 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1668): Closing 1c91405cdf5571de133464d086f5ac7f, disabling compactions & flushes 2023-11-12T09:31:15,812 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1690): Closing region table1,,1699781452630.1c91405cdf5571de133464d086f5ac7f. 2023-11-12T09:31:15,812 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1699781452630.1c91405cdf5571de133464d086f5ac7f. 2023-11-12T09:31:15,812 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1778): Acquired close lock on table1,,1699781452630.1c91405cdf5571de133464d086f5ac7f. after waiting 0 ms 2023-11-12T09:31:15,812 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1788): Updates disabled for region table1,,1699781452630.1c91405cdf5571de133464d086f5ac7f. 2023-11-12T09:31:15,822 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/table1/1c91405cdf5571de133464d086f5ac7f/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-11-12T09:31:15,823 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:31:15,824 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1909): Closed table1,,1699781452630.1c91405cdf5571de133464d086f5ac7f. 2023-11-12T09:31:15,824 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1622): Region close journal for 1c91405cdf5571de133464d086f5ac7f: 2023-11-12T09:31:15,826 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] handler.UnassignRegionHandler(163): Closed 1c91405cdf5571de133464d086f5ac7f 2023-11-12T09:31:15,827 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=51 updating hbase:meta row=1c91405cdf5571de133464d086f5ac7f, regionState=CLOSED 2023-11-12T09:31:15,831 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=52, resume processing ppid=51 2023-11-12T09:31:15,831 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=52, ppid=51, state=SUCCESS; CloseRegionProcedure 1c91405cdf5571de133464d086f5ac7f, server=jenkins-hbase3.apache.org,37573,1699781400943 in 170 msec 2023-11-12T09:31:15,833 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=51, resume processing ppid=50 2023-11-12T09:31:15,833 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=51, ppid=50, state=SUCCESS; TransitRegionStateProcedure table=table1, region=1c91405cdf5571de133464d086f5ac7f, UNASSIGN in 178 msec 2023-11-12T09:31:15,834 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781475833"}]},"ts":"1699781475833"} 2023-11-12T09:31:15,835 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLED in hbase:meta 2023-11-12T09:31:15,837 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set table1 to state=DISABLED 2023-11-12T09:31:15,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=50, state=SUCCESS; DisableTableProcedure table=table1 in 202 msec 2023-11-12T09:31:15,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-12T09:31:15,951 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] client.HBaseAdmin$TableFuture(3745): Operation: DISABLE, Table Name: default:table1, procId: 50 completed 2023-11-12T09:31:15,952 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.HMaster$5(2449): Client=jenkins//172.31.12.81 delete table1 2023-11-12T09:31:15,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] procedure2.ProcedureExecutor(1032): Stored pid=53, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=table1 2023-11-12T09:31:15,956 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=53, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-11-12T09:31:15,957 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=53, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=table1 2023-11-12T09:31:15,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=53 2023-11-12T09:31:15,959 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/table1/1c91405cdf5571de133464d086f5ac7f 2023-11-12T09:31:15,961 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/table1/1c91405cdf5571de133464d086f5ac7f/f, FileablePath, hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/table1/1c91405cdf5571de133464d086f5ac7f/recovered.edits] 2023-11-12T09:31:15,967 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/table1/1c91405cdf5571de133464d086f5ac7f/f/cfff3aa0cca7485eb1b6784ab68b50d7_SeqId_4_ to hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/archive/data/default/table1/1c91405cdf5571de133464d086f5ac7f/f/cfff3aa0cca7485eb1b6784ab68b50d7_SeqId_4_ 2023-11-12T09:31:15,970 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/table1/1c91405cdf5571de133464d086f5ac7f/recovered.edits/6.seqid to hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/archive/data/default/table1/1c91405cdf5571de133464d086f5ac7f/recovered.edits/6.seqid 2023-11-12T09:31:15,971 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/table1/1c91405cdf5571de133464d086f5ac7f 2023-11-12T09:31:15,971 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(293): Archived table1 regions 2023-11-12T09:31:15,973 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=53, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=table1 2023-11-12T09:31:15,975 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of table1 from hbase:meta 2023-11-12T09:31:15,977 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(387): Removing 'table1' descriptor. 2023-11-12T09:31:15,978 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=53, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=table1 2023-11-12T09:31:15,978 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(377): Removing 'table1' from region states. 2023-11-12T09:31:15,978 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1,,1699781452630.1c91405cdf5571de133464d086f5ac7f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1699781475978"}]},"ts":"9223372036854775807"} 2023-11-12T09:31:15,980 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-12T09:31:15,980 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 1c91405cdf5571de133464d086f5ac7f, NAME => 'table1,,1699781452630.1c91405cdf5571de133464d086f5ac7f.', STARTKEY => '', ENDKEY => ''}] 2023-11-12T09:31:15,980 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(381): Marking 'table1' as deleted. 2023-11-12T09:31:15,980 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1699781475980"}]},"ts":"9223372036854775807"} 2023-11-12T09:31:15,982 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1678): Deleted table table1 state from META 2023-11-12T09:31:15,984 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(127): Finished pid=53, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-11-12T09:31:15,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=53, state=SUCCESS; DeleteTableProcedure table=table1 in 32 msec 2023-11-12T09:31:16,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44289 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=53 2023-11-12T09:31:16,058 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] client.HBaseAdmin$TableFuture(3745): Operation: DELETE, Table Name: default:table1, procId: 53 completed 2023-11-12T09:31:16,119 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] hbase.ResourceChecker(175): after: backup.TestRemoteRestore#testFullRestoreRemoteWithAlternateRestoreOutputDir Thread=1312 (was 831) Potentially hanging thread: IPC Server handler 45 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 45643 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2039181213_1 at /127.0.0.1:57336 [Waiting for operation #2] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 12 on default port 36435 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 34 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-60 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: Timer-43 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 25 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Client (405311212) connection to jenkins-hbase3.apache.org/172.31.12.81:42073 from jenkins java.lang.Object.wait(Native Method) org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1053) org.apache.hadoop.ipc.Client$Connection.run(Client.java:1097) Potentially hanging thread: nioEventLoopGroup-14-1 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 26 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 7 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 3 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@44990684[State = -1, empty queue] sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 5 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 45 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 39907 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 36 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 46107 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 39 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/cluster_029c07f3-9135-3e2b-fa26-4408331c19ee/dfs/data/data3/current/BP-814015219-172.31.12.81-1699781448837 java.lang.Thread.sleep(Native Method) org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:205) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 2 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 374513367@qtp-38484575-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:40557 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: IPC Server handler 43 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-49 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 36 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Thread[Thread-1302,5,FailOnTimeoutGroup] java.lang.Thread.sleep(Native Method) org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:694) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O server boss #68 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.jboss.netty.channel.socket.nio.NioServerBoss.select(NioServerBoss.java:163) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.NioServerBoss.run(NioServerBoss.java:42) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DeletionService #1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #46 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 30 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-377-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 41 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: DeletionService #3 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 3 on default port 41823 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #58 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #40 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 16 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 18 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 24 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 6 on default port 36435 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/cluster_029c07f3-9135-3e2b-fa26-4408331c19ee/dfs/data/data1/current/BP-814015219-172.31.12.81-1699781448837 java.lang.Thread.sleep(Native Method) org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:205) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 49 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: DataNode DiskChecker thread 1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 26 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-46 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: Container metrics unregistration java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@50a74c81 java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 19 on default port 41823 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-62 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: Timer-54 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: 413985192@qtp-716068770-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: IPC Server handler 10 on default port 41823 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 42 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@6b1ac5d5 java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 33 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeEditLogRoller@56796731 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeEditLogRoller.run(FSNamesystem.java:3883) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 35 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 39 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 37 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-66 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 17 on default port 36435 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 7 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.hdfs.server.namenode.FSNamesystem$LazyPersistFileScrubber@696c68bc java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.server.namenode.FSNamesystem$LazyPersistFileScrubber.run(FSNamesystem.java:3975) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Listener at jenkins-hbase3.apache.org/39019 java.lang.Thread.dumpThreads(Native Method) java.lang.Thread.getAllStackTraces(Thread.java:1615) org.apache.hadoop.hbase.ResourceCheckerJUnitListener$ThreadResourceAnalyzer.getVal(ResourceCheckerJUnitListener.java:49) org.apache.hadoop.hbase.ResourceChecker.fill(ResourceChecker.java:110) org.apache.hadoop.hbase.ResourceChecker.fillEndings(ResourceChecker.java:104) org.apache.hadoop.hbase.ResourceChecker.end(ResourceChecker.java:206) org.apache.hadoop.hbase.ResourceCheckerJUnitListener.end(ResourceCheckerJUnitListener.java:165) org.apache.hadoop.hbase.ResourceCheckerJUnitListener.testFinished(ResourceCheckerJUnitListener.java:185) org.junit.runner.notification.SynchronizedRunListener.testFinished(SynchronizedRunListener.java:87) org.junit.runner.notification.RunNotifier$9.notifyListener(RunNotifier.java:225) org.junit.runner.notification.RunNotifier$SafeNotifier.run(RunNotifier.java:72) org.junit.runner.notification.RunNotifier.fireTestFinished(RunNotifier.java:222) org.junit.internal.runners.model.EachTestNotifier.fireTestFinished(EachTestNotifier.java:38) org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:372) org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) java.util.concurrent.FutureTask.run(FutureTask.java:266) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: pool-403-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server idle connection scanner for port 36435 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 4 on default port 41823 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeResourceMonitor@7f1b5ca9 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeResourceMonitor.run(FSNamesystem.java:3841) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ForkJoinPool-2-worker-7 sun.misc.Unsafe.park(Native Method) java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1824) java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1693) java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) Potentially hanging thread: New I/O worker #61 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 1 on default port 39019 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2039181213_1 at /127.0.0.1:51264 [Waiting for operation #2] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #43 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-51 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 17 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 41963 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@7c856e57 java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 22 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: PacketResponder: BP-416291311-172.31.12.81-1699781394581:blk_1073741871_1047, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 27 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 5 on default port 36435 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 11 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 1405934259@qtp-204583778-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: IPC Server handler 31 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/cluster_029c07f3-9135-3e2b-fa26-4408331c19ee/dfs/data/data2/current/BP-814015219-172.31.12.81-1699781448837 java.lang.Thread.sleep(Native Method) org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:205) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server idle connection scanner for port 39019 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/cluster_029c07f3-9135-3e2b-fa26-4408331c19ee/dfs/data/data5/current/BP-814015219-172.31.12.81-1699781448837 java.lang.Thread.sleep(Native Method) org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:205) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/cluster_029c07f3-9135-3e2b-fa26-4408331c19ee/dfs/data/data2) java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:627) Potentially hanging thread: IPC Server handler 14 on default port 41823 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 2122158058@qtp-260835180-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@670d472d java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server idle connection scanner for port 45497 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 5 on default port 45643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 18 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #35 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 44 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #48 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-48 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 3 on default port 41963 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: HFileArchiver-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 10 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Client (405311212) connection to localhost/127.0.0.1:45497 from jenkins java.lang.Object.wait(Native Method) org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1053) org.apache.hadoop.ipc.Client$Connection.run(Client.java:1097) Potentially hanging thread: IPC Server handler 12 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-40 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 0 on default port 41823 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-361-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 4 on default port 39907 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 40 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 36 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 41 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 1209643549@qtp-324788938-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: IPC Server handler 38 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 40 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: LeaseRenewer:jenkins@localhost:45497 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:412) org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$600(LeaseRenewer.java:76) org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:308) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 11 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Thread[Thread-1327,5,FailOnTimeoutGroup] java.lang.Thread.sleep(Native Method) org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:694) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server idle connection scanner for port 41823 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 22 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 1902199623@qtp-260835180-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:36367 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: 483391207@qtp-204583778-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:35117 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: nioEventLoopGroup-10-1 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: 1271636449@qtp-100403089-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: IPC Server handler 35 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #63 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 29 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 3 on default port 39019 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #60 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-61 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: New I/O worker #62 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #37 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 19 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 9 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-55 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: Thread[Thread-1337,5,FailOnTimeoutGroup] java.lang.Thread.sleep(Native Method) org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:694) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 38 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 30 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 21 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 29 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 6 on default port 41823 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #38 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 23 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: BP-814015219-172.31.12.81-1699781448837 heartbeating to localhost/127.0.0.1:45497 java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.IncrementalBlockReportManager.waitTillNextIBR(IncrementalBlockReportManager.java:158) org.apache.hadoop.hdfs.server.datanode.BPServiceActor.offerService(BPServiceActor.java:715) org.apache.hadoop.hdfs.server.datanode.BPServiceActor.run(BPServiceActor.java:851) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x2c8ba79b-shared-pool-13 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 38 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 27 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 4 on default port 36435 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 1 on default port 36435 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 6 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-357-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: 1385654365@qtp-324788938-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:41995 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: 1565065093@qtp-959892424-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: IPC Server handler 2 on default port 45101 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 31 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: DeletionService #3 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 49 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-278318368_16 at /127.0.0.1:35664 [Receiving block BP-416291311-172.31.12.81-1699781394581:blk_1073741870_1046] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 15 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 22 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #66 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #57 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x2c8ba79b-shared-pool-16 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/cluster_029c07f3-9135-3e2b-fa26-4408331c19ee/dfs/data/data3) java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:627) Potentially hanging thread: BP-814015219-172.31.12.81-1699781448837 heartbeating to localhost/127.0.0.1:45497 java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.IncrementalBlockReportManager.waitTillNextIBR(IncrementalBlockReportManager.java:158) org.apache.hadoop.hdfs.server.datanode.BPServiceActor.offerService(BPServiceActor.java:715) org.apache.hadoop.hdfs.server.datanode.BPServiceActor.run(BPServiceActor.java:851) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 13 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 12 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.hdfs.server.datanode.DataXceiverServer@45604f0f sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:421) sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:249) sun.nio.ch.ServerSocketAdaptor.accept(ServerSocketAdaptor.java:113) org.apache.hadoop.hdfs.net.TcpPeerServer.accept(TcpPeerServer.java:85) org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.run(DataXceiverServer.java:145) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/cluster_029c07f3-9135-3e2b-fa26-4408331c19ee/dfs/data/data5) java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:627) Potentially hanging thread: New I/O worker #65 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: region-location-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 23 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 15 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 13 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-53 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: pool-354-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 17 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 32 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-65 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 32 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 39019 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #42 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #55 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: pool-349-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 3 on default port 36435 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 29 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 48 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 44 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #67 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #39 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 19 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57951@0x56d1f692 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:328) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$538/410066595.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-47 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 0 on default port 39907 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 9 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 33 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: hconnection-0x2c8ba79b-shared-pool-17 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 4 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: DeletionService #2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 24 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-56 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 19 on default port 36435 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-52 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: org.apache.hadoop.hdfs.server.blockmanagement.PendingReplicationBlocks$PendingReplicationMonitor@2abdb4a2 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.server.blockmanagement.PendingReplicationBlocks$PendingReplicationMonitor.run(PendingReplicationBlocks.java:244) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@73d60fc4[State = -1, empty queue] sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: 1511211404@qtp-100403089-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:37977 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: IPC Server handler 9 on default port 45643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-44 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 8 on default port 36435 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: ContainersLauncher #0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1479845225_16 at /127.0.0.1:59608 [Waiting for operation #8] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 6 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 1 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 18 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 45643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 39019 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 11 on default port 36435 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 34091 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 15 on default port 41823 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 11 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 15 on default port 36435 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: nioEventLoopGroup-12-1 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 1 on default port 45101 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-367-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #41 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 4 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 31 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/cluster_029c07f3-9135-3e2b-fa26-4408331c19ee/dfs/data/data4) java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:627) Potentially hanging thread: IPC Server handler 15 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 1 on default port 41963 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 43 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-67 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 1 on default port 39907 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 39907 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: pool-351-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #59 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 18 on default port 41823 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #53 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 39 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: ForkJoinPool-2-worker-6 sun.misc.Unsafe.park(Native Method) java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1824) java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1693) java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) Potentially hanging thread: IPC Server handler 10 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 45101 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-58 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 16 on default port 41823 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: PacketResponder: BP-416291311-172.31.12.81-1699781394581:blk_1073741870_1046, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 12 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: DeletionService #1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: CacheReplicationMonitor(495643056) sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) org.apache.hadoop.hdfs.server.blockmanagement.CacheReplicationMonitor.run(CacheReplicationMonitor.java:181) Potentially hanging thread: IPC Server handler 6 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 3 on default port 45101 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: hconnection-0x2c8ba79b-shared-pool-15 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: pool-371-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 21 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 28 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 3 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/cluster_029c07f3-9135-3e2b-fa26-4408331c19ee/dfs/data/data6) java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:627) Potentially hanging thread: IPC Server handler 9 on default port 36435 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-63 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: Timer-50 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 32 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-64 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: Thread-1344 java.lang.Thread.sleep(Native Method) org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:143) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: org.apache.hadoop.hdfs.server.datanode.DataXceiverServer@12cbba88 sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:421) sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:249) sun.nio.ch.ServerSocketAdaptor.accept(ServerSocketAdaptor.java:113) org.apache.hadoop.hdfs.net.TcpPeerServer.accept(TcpPeerServer.java:85) org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.run(DataXceiverServer.java:145) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-278318368_16 at /127.0.0.1:35678 [Receiving block BP-416291311-172.31.12.81-1699781394581:blk_1073741871_1047] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 5 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 34 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x2c8ba79b-shared-pool-18 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ContainersLauncher #1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-45 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: region-location-3 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 7 on default port 41823 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 38021 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-39 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 45 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 47 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/cluster_029c07f3-9135-3e2b-fa26-4408331c19ee/dfs/data/data4/current/BP-814015219-172.31.12.81-1699781448837 java.lang.Thread.sleep(Native Method) org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:205) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: 2011489108@qtp-1409692604-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:43013 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: IPC Server handler 2 on default port 46107 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 26 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@536ae431 java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ApplicationMasterLauncher #0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 8 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: BP-814015219-172.31.12.81-1699781448837 heartbeating to localhost/127.0.0.1:45497 java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.IncrementalBlockReportManager.waitTillNextIBR(IncrementalBlockReportManager.java:158) org.apache.hadoop.hdfs.server.datanode.BPServiceActor.offerService(BPServiceActor.java:715) org.apache.hadoop.hdfs.server.datanode.BPServiceActor.run(BPServiceActor.java:851) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 14 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor@12ef539f java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor.run(HeartbeatManager.java:536) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Client (405311212) connection to localhost/127.0.0.1:45497 from jenkins java.lang.Object.wait(Native Method) org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1053) org.apache.hadoop.ipc.Client$Connection.run(Client.java:1097) Potentially hanging thread: IPC Server handler 3 on default port 45497 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: DeletionService #0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 14 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 7 on default port 36435 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 42 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@85251cd java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 4 on default port 45643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 38021 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 11 on default port 41823 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: MutableQuantiles-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 0 on default port 45497 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 13 on default port 36435 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #54 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 23 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 20 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-42 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server idle connection scanner for port 46107 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 47 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 41823 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 47 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 13 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 35 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 37 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 43 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 3 on default port 46107 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O server boss #51 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.jboss.netty.channel.socket.nio.NioServerBoss.select(NioServerBoss.java:163) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.NioServerBoss.run(NioServerBoss.java:42) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 37 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 14 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57951@0x56d1f692-SendThread(127.0.0.1:57951) sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: IPC Server handler 0 on default port 41963 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/cluster_029c07f3-9135-3e2b-fa26-4408331c19ee/dfs/data/data1) java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:627) Potentially hanging thread: IPC Server handler 4 on default port 41963 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 45497 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 46 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 16 on default port 36435 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 7 on default port 45643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 28 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 8 on default port 45643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 40 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 12 on default port 41823 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-400-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57951@0x56d1f692-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:549) Potentially hanging thread: IPC Server handler 4 on default port 39019 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 25 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 8 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 21 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 9 on default port 41823 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 19 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 44031 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: Timer-41 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 6 on default port 45643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 5 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: 1102269516@qtp-1409692604-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: Timer-57 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 0 on default port 36435 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 41 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 3 on default port 39907 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 1 on default port 46107 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #36 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Client (405311212) connection to jenkins-hbase3.apache.org/172.31.12.81:39019 from jenkins java.lang.Object.wait(Native Method) org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1053) org.apache.hadoop.ipc.Client$Connection.run(Client.java:1097) Potentially hanging thread: IPC Server handler 46 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 10 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 46 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-356-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #56 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #52 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@a278bdb java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-59 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 20 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: hconnection-0x2c8ba79b-shared-pool-14 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 4 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-405-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 1 on default port 41823 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor@90f4485 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor.run(LeaseManager.java:533) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #47 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 3 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 7 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 4 on default port 46107 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 44 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 16 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 25 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 48 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #49 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 0 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 5 on default port 41823 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 9 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 49 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #50 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@285a93ec java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 18 on default port 36435 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 612467593@qtp-959892424-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:43279 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: pool-401-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server idle connection scanner for port 45101 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: ForkJoinPool-2-worker-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1824) java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1693) java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) Potentially hanging thread: IPC Server handler 13 on default port 41823 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #44 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 4 on default port 45101 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 4 on default port 45497 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: DeletionService #2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server idle connection scanner for port 34091 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a73ff986-57fc-cb93-0ef2-ac6ee95016fb/cluster_029c07f3-9135-3e2b-fa26-4408331c19ee/dfs/data/data6/current/BP-814015219-172.31.12.81-1699781448837 java.lang.Thread.sleep(Native Method) org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:205) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 8 on default port 41823 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 45643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 35061 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 28 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 20 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 1 on default port 45643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 41963 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 33 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #45 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 24 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 17 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 8 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 3 on default port 45643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 17 on default port 41823 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-373-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 2 on default port 36435 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 27 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 14 on default port 36435 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@1eea8b91[State = -1, empty queue] sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #64 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Thread-2052 java.io.FileInputStream.readBytes(Native Method) java.io.FileInputStream.read(FileInputStream.java:255) java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.io.BufferedInputStream.read(BufferedInputStream.java:345) sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:284) sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:326) sun.nio.cs.StreamDecoder.read(StreamDecoder.java:178) java.io.InputStreamReader.read(InputStreamReader.java:184) java.io.BufferedReader.fill(BufferedReader.java:161) java.io.BufferedReader.readLine(BufferedReader.java:324) java.io.BufferedReader.readLine(BufferedReader.java:389) org.apache.hadoop.util.Shell$1.run(Shell.java:955) Potentially hanging thread: IPC Server handler 30 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: ContainersLauncher #0 java.io.FileInputStream.readBytes(Native Method) java.io.FileInputStream.read(FileInputStream.java:255) java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.io.BufferedInputStream.read(BufferedInputStream.java:345) sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:284) sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:326) sun.nio.cs.StreamDecoder.read(StreamDecoder.java:178) java.io.InputStreamReader.read(InputStreamReader.java:184) java.io.BufferedReader.fill(BufferedReader.java:161) java.io.BufferedReader.read1(BufferedReader.java:212) java.io.BufferedReader.read(BufferedReader.java:286) org.apache.hadoop.util.Shell$ShellCommandExecutor.parseExecResult(Shell.java:1229) org.apache.hadoop.util.Shell.runCommand(Shell.java:984) org.apache.hadoop.util.Shell.run(Shell.java:884) org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1216) org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor.launchContainer(DefaultContainerExecutor.java:294) org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.launchContainer(ContainerLaunch.java:447) org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.call(ContainerLaunch.java:298) org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.call(ContainerLaunch.java:99) java.util.concurrent.FutureTask.run(FutureTask.java:266) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server idle connection scanner for port 43643 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 16 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: region-location-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 1 on default port 45497 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 1 on default port 44031 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-363-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 34 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 1710447152@qtp-716068770-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:37751 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: DeletionService #0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 1 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 10 on default port 36435 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 1270770795@qtp-38484575-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: org.apache.hadoop.hdfs.server.datanode.DataXceiverServer@63587622 sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:421) sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:249) sun.nio.ch.ServerSocketAdaptor.accept(ServerSocketAdaptor.java:113) org.apache.hadoop.hdfs.net.TcpPeerServer.accept(TcpPeerServer.java:85) org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.run(DataXceiverServer.java:145) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 48 on default port 43643 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 42 on default port 35061 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) - Thread LEAK? -, OpenFileDescriptor=1293 (was 935) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=291 (was 183) - SystemLoadAverage LEAK? -, ProcessCount=171 (was 169) - ProcessCount LEAK? -, AvailableMemoryMB=5475 (was 5365) - AvailableMemoryMB LEAK? - 2023-11-12T09:31:16,122 WARN [Listener at jenkins-hbase3.apache.org/39019 {}] hbase.ResourceChecker(130): Thread=1312 is superior to 500 2023-11-12T09:31:16,122 WARN [Listener at jenkins-hbase3.apache.org/39019 {}] hbase.ResourceChecker(130): OpenFileDescriptor=1293 is superior to 1024 2023-11-12T09:31:16,129 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-11-12T09:31:16,130 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x66e0d3e8 to 127.0.0.1:57951 2023-11-12T09:31:16,130 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:31:16,136 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-11-12T09:31:16,136 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] util.JVMClusterUtil(257): Found active master hash=1366801590, stopped=false 2023-11-12T09:31:16,136 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] master.ServerManager(896): Cluster shutdown requested of master=jenkins-hbase3.apache.org,37645,1699781405833 2023-11-12T09:31:16,138 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/2/running 2023-11-12T09:31:16,138 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] procedure2.ProcedureExecutor(630): Stopping 2023-11-12T09:31:16,138 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-12T09:31:16,138 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33205-0x1004cf7b7840005, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/2/running 2023-11-12T09:31:16,138 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33205-0x1004cf7b7840005, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-12T09:31:16,138 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:31:16,139 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,33205,1699781406053' ***** 2023-11-12T09:31:16,139 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] regionserver.HRegionServer(2531): STOPPED: Shutdown requested 2023-11-12T09:31:16,139 INFO [RS:0;jenkins-hbase3:33205 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-11-12T09:31:16,139 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33205-0x1004cf7b7840005, quorum=127.0.0.1:57951, baseZNode=/2 Set watcher on znode that does not yet exist, /2/running 2023-11-12T09:31:16,141 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Set watcher on znode that does not yet exist, /2/running 2023-11-12T09:31:16,161 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-11-12T09:31:16,162 INFO [RS:0;jenkins-hbase3:33205 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@5d4d826a{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-11-12T09:31:16,166 INFO [RS:0;jenkins-hbase3:33205 {}] server.AbstractConnector(383): Stopped ServerConnector@4ee83247{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-11-12T09:31:16,166 INFO [RS:0;jenkins-hbase3:33205 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-11-12T09:31:16,168 INFO [RS:0;jenkins-hbase3:33205 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@588f8551{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-11-12T09:31:16,169 INFO [RS:0;jenkins-hbase3:33205 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@1d942304{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/hadoop.log.dir/,STOPPED} 2023-11-12T09:31:16,171 INFO [RS:0;jenkins-hbase3:33205 {}] regionserver.HeapMemoryManager(220): Stopping 2023-11-12T09:31:16,172 INFO [RS:0;jenkins-hbase3:33205 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2023-11-12T09:31:16,172 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-11-12T09:31:16,172 INFO [RS:0;jenkins-hbase3:33205 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-11-12T09:31:16,172 INFO [RS:0;jenkins-hbase3:33205 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-11-12T09:31:16,172 INFO [RS:0;jenkins-hbase3:33205 {}] regionserver.HRegionServer(3528): Received CLOSE for 3e47234fadef92f1114692ec74b0ca02 2023-11-12T09:31:16,173 INFO [RS:0;jenkins-hbase3:33205 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,33205,1699781406053 2023-11-12T09:31:16,173 DEBUG [RS:0;jenkins-hbase3:33205 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:31:16,174 INFO [RS:0;jenkins-hbase3:33205 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-11-12T09:31:16,174 INFO [RS:0;jenkins-hbase3:33205 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-11-12T09:31:16,174 INFO [RS:0;jenkins-hbase3:33205 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-11-12T09:31:16,174 INFO [RS:0;jenkins-hbase3:33205 {}] regionserver.HRegionServer(3528): Received CLOSE for 1588230740 2023-11-12T09:31:16,175 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 3e47234fadef92f1114692ec74b0ca02, disabling compactions & flushes 2023-11-12T09:31:16,175 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1699781408087.3e47234fadef92f1114692ec74b0ca02. 2023-11-12T09:31:16,175 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1699781408087.3e47234fadef92f1114692ec74b0ca02. 2023-11-12T09:31:16,175 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1699781408087.3e47234fadef92f1114692ec74b0ca02. after waiting 0 ms 2023-11-12T09:31:16,176 INFO [RS:0;jenkins-hbase3:33205 {}] regionserver.HRegionServer(1579): Waiting on 2 regions to close 2023-11-12T09:31:16,176 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1699781408087.3e47234fadef92f1114692ec74b0ca02. 2023-11-12T09:31:16,176 DEBUG [RS:0;jenkins-hbase3:33205 {}] regionserver.HRegionServer(1583): Online Regions={3e47234fadef92f1114692ec74b0ca02=hbase:namespace,,1699781408087.3e47234fadef92f1114692ec74b0ca02., 1588230740=hbase:meta,,1.1588230740} 2023-11-12T09:31:16,176 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 3e47234fadef92f1114692ec74b0ca02 1/1 column families, dataSize=78 B heapSize=488 B 2023-11-12T09:31:16,179 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-11-12T09:31:16,179 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-11-12T09:31:16,179 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-11-12T09:31:16,179 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-11-12T09:31:16,179 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-11-12T09:31:16,179 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=1.26 KB heapSize=2.89 KB 2023-11-12T09:31:16,179 DEBUG [RS:0;jenkins-hbase3:33205 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 3e47234fadef92f1114692ec74b0ca02 2023-11-12T09:31:16,205 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/data/hbase/namespace/3e47234fadef92f1114692ec74b0ca02/.tmp/info/f859aaca107249fab877c96ab9eea80d is 45, key is default/info:d/1699781409030/Put/seqid=0 2023-11-12T09:31:16,206 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/data/hbase/meta/1588230740/.tmp/info/91623945ccd24bb9ae8db9a0776fd898 is 143, key is hbase:namespace,,1699781408087.3e47234fadef92f1114692ec74b0ca02./info:regioninfo/1699781408947/Put/seqid=0 2023-11-12T09:31:16,213 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.17 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/data/hbase/meta/1588230740/.tmp/info/91623945ccd24bb9ae8db9a0776fd898 2023-11-12T09:31:16,243 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/data/hbase/meta/1588230740/.tmp/table/acadc2d6501e481a8cb131224fae4eca is 51, key is hbase:namespace/table:state/1699781408970/Put/seqid=0 2023-11-12T09:31:16,380 DEBUG [RS:0;jenkins-hbase3:33205 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 3e47234fadef92f1114692ec74b0ca02 2023-11-12T09:31:16,420 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-11-12T09:31:16,420 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-11-12T09:31:16,580 DEBUG [RS:0;jenkins-hbase3:33205 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 3e47234fadef92f1114692ec74b0ca02 2023-11-12T09:31:16,613 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/data/hbase/namespace/3e47234fadef92f1114692ec74b0ca02/.tmp/info/f859aaca107249fab877c96ab9eea80d 2023-11-12T09:31:16,622 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/data/hbase/namespace/3e47234fadef92f1114692ec74b0ca02/.tmp/info/f859aaca107249fab877c96ab9eea80d as hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/data/hbase/namespace/3e47234fadef92f1114692ec74b0ca02/info/f859aaca107249fab877c96ab9eea80d 2023-11-12T09:31:16,627 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/data/hbase/namespace/3e47234fadef92f1114692ec74b0ca02/info/f859aaca107249fab877c96ab9eea80d, entries=2, sequenceid=6, filesize=4.9 K 2023-11-12T09:31:16,628 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 3e47234fadef92f1114692ec74b0ca02 in 452ms, sequenceid=6, compaction requested=false 2023-11-12T09:31:16,638 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41663/tmp/wal/data/hbase/namespace/3e47234fadef92f1114692ec74b0ca02/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2023-11-12T09:31:16,638 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:31:16,641 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1699781408087.3e47234fadef92f1114692ec74b0ca02. 2023-11-12T09:31:16,641 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 3e47234fadef92f1114692ec74b0ca02: 2023-11-12T09:31:16,641 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1699781408087.3e47234fadef92f1114692ec74b0ca02. 2023-11-12T09:31:16,647 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=94 B at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/data/hbase/meta/1588230740/.tmp/table/acadc2d6501e481a8cb131224fae4eca 2023-11-12T09:31:16,652 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/data/hbase/meta/1588230740/.tmp/info/91623945ccd24bb9ae8db9a0776fd898 as hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/data/hbase/meta/1588230740/info/91623945ccd24bb9ae8db9a0776fd898 2023-11-12T09:31:16,658 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/data/hbase/meta/1588230740/info/91623945ccd24bb9ae8db9a0776fd898, entries=10, sequenceid=9, filesize=6.4 K 2023-11-12T09:31:16,659 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/data/hbase/meta/1588230740/.tmp/table/acadc2d6501e481a8cb131224fae4eca as hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/data/hbase/meta/1588230740/table/acadc2d6501e481a8cb131224fae4eca 2023-11-12T09:31:16,665 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/data/hbase/meta/1588230740/table/acadc2d6501e481a8cb131224fae4eca, entries=2, sequenceid=9, filesize=5.1 K 2023-11-12T09:31:16,666 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~1.26 KB/1290, heapSize ~2.61 KB/2672, currentSize=0 B/0 for 1588230740 in 487ms, sequenceid=9, compaction requested=false 2023-11-12T09:31:16,681 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41663/tmp/wal/data/hbase/meta/1588230740/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2023-11-12T09:31:16,682 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:31:16,682 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-11-12T09:31:16,682 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-11-12T09:31:16,682 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-11-12T09:31:16,682 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-11-12T09:31:16,780 INFO [RS:0;jenkins-hbase3:33205 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,33205,1699781406053; all regions closed. 2023-11-12T09:31:16,789 DEBUG [RS:0;jenkins-hbase3:33205 {}] wal.AbstractFSWAL(1062): Moved 1 WAL file(s) to /tmp/wal/oldWALs 2023-11-12T09:31:16,789 INFO [RS:0;jenkins-hbase3:33205 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C33205%2C1699781406053.meta:.meta(num 1699781407967) 2023-11-12T09:31:16,795 DEBUG [RS:0;jenkins-hbase3:33205 {}] wal.AbstractFSWAL(1062): Moved 1 WAL file(s) to /tmp/wal/oldWALs 2023-11-12T09:31:16,795 INFO [RS:0;jenkins-hbase3:33205 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C33205%2C1699781406053:(num 1699781407545) 2023-11-12T09:31:16,796 DEBUG [RS:0;jenkins-hbase3:33205 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:31:16,796 INFO [RS:0;jenkins-hbase3:33205 {}] regionserver.LeaseManager(133): Closed leases 2023-11-12T09:31:16,796 INFO [RS:0;jenkins-hbase3:33205 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2023-11-12T09:31:16,796 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-11-12T09:31:16,799 INFO [RS:0;jenkins-hbase3:33205 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:33205 2023-11-12T09:31:16,807 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/rs 2023-11-12T09:31:16,807 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33205-0x1004cf7b7840005, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/2/rs/jenkins-hbase3.apache.org,33205,1699781406053 2023-11-12T09:31:16,810 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,33205,1699781406053] 2023-11-12T09:31:16,810 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,33205,1699781406053; numProcessing=1 2023-11-12T09:31:16,812 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(224): Node /2/draining/jenkins-hbase3.apache.org,33205,1699781406053 already deleted, retry=false 2023-11-12T09:31:16,813 INFO [RegionServerTracker-0 {}] master.ServerManager(561): Cluster shutdown set; jenkins-hbase3.apache.org,33205,1699781406053 expired; onlineServers=0 2023-11-12T09:31:16,813 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,37645,1699781405833' ***** 2023-11-12T09:31:16,813 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2531): STOPPED: Cluster shutdown set; onlineServer=0 2023-11-12T09:31:16,815 DEBUG [M:0;jenkins-hbase3:37645 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@375d71f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-11-12T09:31:16,815 INFO [M:0;jenkins-hbase3:37645 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-11-12T09:31:16,832 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/2/master 2023-11-12T09:31:16,832 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-12T09:31:16,834 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Set watcher on znode that does not yet exist, /2/master 2023-11-12T09:31:16,923 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33205-0x1004cf7b7840005, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-11-12T09:31:16,923 INFO [RS:0;jenkins-hbase3:33205 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,33205,1699781406053; zookeeper connection closed. 2023-11-12T09:31:16,923 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33205-0x1004cf7b7840005, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-11-12T09:31:16,927 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1649989 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1649989 2023-11-12T09:31:16,928 INFO [M:0;jenkins-hbase3:37645 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@28e08836{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-11-12T09:31:16,928 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-11-12T09:31:16,928 INFO [M:0;jenkins-hbase3:37645 {}] server.AbstractConnector(383): Stopped ServerConnector@10166c60{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-11-12T09:31:16,928 INFO [M:0;jenkins-hbase3:37645 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-11-12T09:31:16,929 INFO [M:0;jenkins-hbase3:37645 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@6407dde3{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-11-12T09:31:16,931 INFO [M:0;jenkins-hbase3:37645 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@45b38ffb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/hadoop.log.dir/,STOPPED} 2023-11-12T09:31:16,931 INFO [M:0;jenkins-hbase3:37645 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,37645,1699781405833 2023-11-12T09:31:16,931 INFO [M:0;jenkins-hbase3:37645 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,37645,1699781405833; all regions closed. 2023-11-12T09:31:16,931 DEBUG [M:0;jenkins-hbase3:37645 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:31:16,931 INFO [M:0;jenkins-hbase3:37645 {}] master.HMaster(1666): Stopping master jetty server 2023-11-12T09:31:16,932 INFO [M:0;jenkins-hbase3:37645 {}] server.AbstractConnector(383): Stopped ServerConnector@76f62d72{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-11-12T09:31:16,932 DEBUG [M:0;jenkins-hbase3:37645 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-11-12T09:31:16,933 INFO [M:0;jenkins-hbase3:37645 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-11-12T09:31:16,933 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-11-12T09:31:16,933 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1699781407265 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1699781407265,5,FailOnTimeoutGroup] 2023-11-12T09:31:16,933 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1699781407265 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1699781407265,5,FailOnTimeoutGroup] 2023-11-12T09:31:16,933 DEBUG [M:0;jenkins-hbase3:37645 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x43dfa29b to 127.0.0.1:57951 2023-11-12T09:31:16,933 DEBUG [M:0;jenkins-hbase3:37645 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:31:16,933 DEBUG [M:0;jenkins-hbase3:37645 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-11-12T09:31:16,933 INFO [M:0;jenkins-hbase3:37645 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-11-12T09:31:16,933 DEBUG [M:0;jenkins-hbase3:37645 {}] master.HMaster(1689): Stopping service threads 2023-11-12T09:31:16,934 INFO [M:0;jenkins-hbase3:37645 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-11-12T09:31:16,934 ERROR [M:0;jenkins-hbase3:37645 {}] procedure2.ProcedureExecutor(654): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] 2023-11-12T09:31:16,934 INFO [M:0;jenkins-hbase3:37645 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-11-12T09:31:16,934 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-11-12T09:31:16,935 DEBUG [M:0;jenkins-hbase3:37645 {}] zookeeper.ZKUtil(347): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Unable to get data of znode /2/master because node does not exist (not an error) 2023-11-12T09:31:16,935 WARN [M:0;jenkins-hbase3:37645 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-11-12T09:31:16,935 INFO [M:0;jenkins-hbase3:37645 {}] assignment.AssignmentManager(383): Stopping assignment manager 2023-11-12T09:31:16,936 INFO [M:0;jenkins-hbase3:37645 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-11-12T09:31:16,937 DEBUG [M:0;jenkins-hbase3:37645 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-11-12T09:31:16,953 INFO [M:0;jenkins-hbase3:37645 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:31:16,953 DEBUG [M:0;jenkins-hbase3:37645 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:31:16,953 DEBUG [M:0;jenkins-hbase3:37645 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-11-12T09:31:16,953 DEBUG [M:0;jenkins-hbase3:37645 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:31:16,953 INFO [M:0;jenkins-hbase3:37645 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=25.76 KB heapSize=33.30 KB 2023-11-12T09:31:16,967 DEBUG [M:0;jenkins-hbase3:37645 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cb7c5e67ae1a40158492d15040135da6 is 88, key is hbase:meta,,1/info:sn/1699781407787/Put/seqid=0 2023-11-12T09:31:17,372 INFO [M:0;jenkins-hbase3:37645 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cb7c5e67ae1a40158492d15040135da6 2023-11-12T09:31:17,392 DEBUG [M:0;jenkins-hbase3:37645 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/01bfece5818844f1afa87b17d59dd1aa is 241, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1699781408064/Put/seqid=0 2023-11-12T09:31:17,797 INFO [M:0;jenkins-hbase3:37645 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25.17 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/01bfece5818844f1afa87b17d59dd1aa 2023-11-12T09:31:17,804 INFO [M:0;jenkins-hbase3:37645 {}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 01bfece5818844f1afa87b17d59dd1aa 2023-11-12T09:31:17,817 DEBUG [M:0;jenkins-hbase3:37645 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cf1911691b68406e97d889be4f01c9f2 is 82, key is jenkins-hbase3.apache.org,33205,1699781406053/rs:state/1699781407315/Put/seqid=0 2023-11-12T09:31:18,222 INFO [M:0;jenkins-hbase3:37645 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cf1911691b68406e97d889be4f01c9f2 2023-11-12T09:31:18,228 DEBUG [M:0;jenkins-hbase3:37645 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cb7c5e67ae1a40158492d15040135da6 as hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cb7c5e67ae1a40158492d15040135da6 2023-11-12T09:31:18,234 INFO [M:0;jenkins-hbase3:37645 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cb7c5e67ae1a40158492d15040135da6, entries=8, sequenceid=76, filesize=5.5 K 2023-11-12T09:31:18,235 DEBUG [M:0;jenkins-hbase3:37645 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/01bfece5818844f1afa87b17d59dd1aa as hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/01bfece5818844f1afa87b17d59dd1aa 2023-11-12T09:31:18,240 INFO [M:0;jenkins-hbase3:37645 {}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 01bfece5818844f1afa87b17d59dd1aa 2023-11-12T09:31:18,240 INFO [M:0;jenkins-hbase3:37645 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/01bfece5818844f1afa87b17d59dd1aa, entries=8, sequenceid=76, filesize=5.4 K 2023-11-12T09:31:18,241 DEBUG [M:0;jenkins-hbase3:37645 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cf1911691b68406e97d889be4f01c9f2 as hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cf1911691b68406e97d889be4f01c9f2 2023-11-12T09:31:18,246 INFO [M:0;jenkins-hbase3:37645 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41663/user/jenkins/test-data/b2ff921c-8da9-d40a-af0a-c25369647ff1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cf1911691b68406e97d889be4f01c9f2, entries=1, sequenceid=76, filesize=5.1 K 2023-11-12T09:31:18,247 INFO [M:0;jenkins-hbase3:37645 {}] regionserver.HRegion(3022): Finished flush of dataSize ~25.76 KB/26381, heapSize ~33.01 KB/33800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1294ms, sequenceid=76, compaction requested=false 2023-11-12T09:31:18,257 INFO [M:0;jenkins-hbase3:37645 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:31:18,257 DEBUG [M:0;jenkins-hbase3:37645 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-11-12T09:31:18,261 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-11-12T09:31:18,261 INFO [M:0;jenkins-hbase3:37645 {}] flush.MasterFlushTableProcedureManager(85): stop: server shutting down. 2023-11-12T09:31:18,261 INFO [M:0;jenkins-hbase3:37645 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-11-12T09:31:18,263 INFO [M:0;jenkins-hbase3:37645 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:37645 2023-11-12T09:31:18,265 DEBUG [M:0;jenkins-hbase3:37645 {}] zookeeper.RecoverableZooKeeper(224): Node /2/rs/jenkins-hbase3.apache.org,37645,1699781405833 already deleted, retry=false 2023-11-12T09:31:18,368 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-11-12T09:31:18,368 INFO [M:0;jenkins-hbase3:37645 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,37645,1699781405833; zookeeper connection closed. 2023-11-12T09:31:18,368 DEBUG [Listener at localhost/36745-EventThread {}] zookeeper.ZKWatcher(604): master:37645-0x1004cf7b7840004, quorum=127.0.0.1:57951, baseZNode=/2 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-11-12T09:31:18,372 WARN [Listener at jenkins-hbase3.apache.org/39019 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-11-12T09:31:18,382 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-11-12T09:31:18,490 WARN [BP-285580598-172.31.12.81-1699781404724 heartbeating to localhost/127.0.0.1:41663 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-11-12T09:31:18,490 WARN [BP-285580598-172.31.12.81-1699781404724 heartbeating to localhost/127.0.0.1:41663 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-285580598-172.31.12.81-1699781404724 (Datanode Uuid a8473d81-fbfe-47b9-a243-4f97ea75cc63) service to localhost/127.0.0.1:41663 2023-11-12T09:31:18,492 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/cluster_737aeac4-7599-9d23-aa4c-06ede0824ece/dfs/data/data5/current/BP-285580598-172.31.12.81-1699781404724 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-11-12T09:31:18,492 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/cluster_737aeac4-7599-9d23-aa4c-06ede0824ece/dfs/data/data6/current/BP-285580598-172.31.12.81-1699781404724 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-11-12T09:31:18,498 WARN [Listener at jenkins-hbase3.apache.org/39019 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-11-12T09:31:18,507 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-11-12T09:31:18,615 WARN [BP-285580598-172.31.12.81-1699781404724 heartbeating to localhost/127.0.0.1:41663 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-11-12T09:31:18,615 WARN [BP-285580598-172.31.12.81-1699781404724 heartbeating to localhost/127.0.0.1:41663 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-285580598-172.31.12.81-1699781404724 (Datanode Uuid 9976839f-8359-402e-861c-6a253624354b) service to localhost/127.0.0.1:41663 2023-11-12T09:31:18,616 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/cluster_737aeac4-7599-9d23-aa4c-06ede0824ece/dfs/data/data3/current/BP-285580598-172.31.12.81-1699781404724 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-11-12T09:31:18,617 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/cluster_737aeac4-7599-9d23-aa4c-06ede0824ece/dfs/data/data4/current/BP-285580598-172.31.12.81-1699781404724 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-11-12T09:31:18,623 WARN [Listener at jenkins-hbase3.apache.org/39019 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-11-12T09:31:18,634 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-11-12T09:31:18,749 WARN [BP-285580598-172.31.12.81-1699781404724 heartbeating to localhost/127.0.0.1:41663 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-11-12T09:31:18,749 WARN [BP-285580598-172.31.12.81-1699781404724 heartbeating to localhost/127.0.0.1:41663 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-285580598-172.31.12.81-1699781404724 (Datanode Uuid 47f26c7d-43a3-46f0-91e1-438554372551) service to localhost/127.0.0.1:41663 2023-11-12T09:31:18,749 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/cluster_737aeac4-7599-9d23-aa4c-06ede0824ece/dfs/data/data1/current/BP-285580598-172.31.12.81-1699781404724 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-11-12T09:31:18,750 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3d0ba3b8-3fa4-1086-81cf-fa0d67d9e303/cluster_737aeac4-7599-9d23-aa4c-06ede0824ece/dfs/data/data2/current/BP-285580598-172.31.12.81-1699781404724 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-11-12T09:31:18,803 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-11-12T09:31:18,915 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2023-11-12T09:31:18,950 INFO [Finalizer {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-12T09:31:18,950 DEBUG [Finalizer {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x56d1f692 to 127.0.0.1:57951 2023-11-12T09:31:18,950 DEBUG [Finalizer {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:31:18,985 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-11-12T09:31:18,985 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-11-12T09:31:18,985 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-12T09:31:18,985 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x196df6a0 to 127.0.0.1:57951 2023-11-12T09:31:18,985 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:31:18,985 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-11-12T09:31:18,985 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] util.JVMClusterUtil(257): Found active master hash=1814422580, stopped=false 2023-11-12T09:31:18,985 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] master.ServerManager(896): Cluster shutdown requested of master=jenkins-hbase3.apache.org,44289,1699781397811 2023-11-12T09:31:18,988 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2023-11-12T09:31:18,988 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2023-11-12T09:31:18,988 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:31:18,988 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:31:18,988 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] procedure2.ProcedureExecutor(630): Stopping 2023-11-12T09:31:18,988 DEBUG [Listener at jenkins-hbase3.apache.org/39019 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:31:18,988 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,37573,1699781400943' ***** 2023-11-12T09:31:18,988 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] regionserver.HRegionServer(2531): STOPPED: Shutdown requested 2023-11-12T09:31:18,988 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-11-12T09:31:18,988 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-11-12T09:31:18,988 INFO [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-11-12T09:31:18,989 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-11-12T09:31:18,996 INFO [RS:0;jenkins-hbase3:37573 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@559a367f{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-11-12T09:31:18,996 INFO [RS:0;jenkins-hbase3:37573 {}] server.AbstractConnector(383): Stopped ServerConnector@220910c3{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-11-12T09:31:18,997 INFO [RS:0;jenkins-hbase3:37573 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-11-12T09:31:18,998 INFO [RS:0;jenkins-hbase3:37573 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@5892ebca{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-11-12T09:31:18,999 INFO [RS:0;jenkins-hbase3:37573 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@be88332{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/hadoop.log.dir/,STOPPED} 2023-11-12T09:31:19,000 INFO [RS:0;jenkins-hbase3:37573 {}] regionserver.HeapMemoryManager(220): Stopping 2023-11-12T09:31:19,000 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-11-12T09:31:19,000 INFO [RS:0;jenkins-hbase3:37573 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2023-11-12T09:31:19,000 INFO [RS:0;jenkins-hbase3:37573 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-11-12T09:31:19,000 INFO [RS:0;jenkins-hbase3:37573 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-11-12T09:31:19,000 INFO [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(3528): Received CLOSE for e2e74d85dcb767a4afcaf7637dfbf1b3 2023-11-12T09:31:19,000 INFO [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(3528): Received CLOSE for daff3e7fcafd33ff24024100cbc1e9c9 2023-11-12T09:31:19,000 INFO [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(3528): Received CLOSE for e4f203e272def65afc0882551ad2e3de 2023-11-12T09:31:19,000 INFO [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(3528): Received CLOSE for 37c7f0bb858c8098fd397daa8ed9cc98 2023-11-12T09:31:19,001 INFO [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(3528): Received CLOSE for 7dc83426edfa8b37518a7f14a9649f4c 2023-11-12T09:31:19,001 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing e2e74d85dcb767a4afcaf7637dfbf1b3, disabling compactions & flushes 2023-11-12T09:31:19,001 INFO [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(3528): Received CLOSE for f85d069e361064154880bd8654ab12c0 2023-11-12T09:31:19,001 INFO [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(3528): Received CLOSE for 441fc40990b285ba0c11d5ae2272b2af 2023-11-12T09:31:19,001 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns3:test-16997814143202,,1699781417816.e2e74d85dcb767a4afcaf7637dfbf1b3. 2023-11-12T09:31:19,001 INFO [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:31:19,002 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16997814143202,,1699781417816.e2e74d85dcb767a4afcaf7637dfbf1b3. 2023-11-12T09:31:19,002 INFO [RS:0;jenkins-hbase3:37573 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-12T09:31:19,002 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16997814143202,,1699781417816.e2e74d85dcb767a4afcaf7637dfbf1b3. after waiting 0 ms 2023-11-12T09:31:19,002 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16997814143202,,1699781417816.e2e74d85dcb767a4afcaf7637dfbf1b3. 2023-11-12T09:31:19,002 DEBUG [RS:0;jenkins-hbase3:37573 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:31:19,002 INFO [RS:0;jenkins-hbase3:37573 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-11-12T09:31:19,002 INFO [RS:0;jenkins-hbase3:37573 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-11-12T09:31:19,002 INFO [RS:0;jenkins-hbase3:37573 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-11-12T09:31:19,002 INFO [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(3528): Received CLOSE for 1588230740 2023-11-12T09:31:19,002 INFO [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(1579): Waiting on 8 regions to close 2023-11-12T09:31:19,002 DEBUG [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(1583): Online Regions={e2e74d85dcb767a4afcaf7637dfbf1b3=ns3:test-16997814143202,,1699781417816.e2e74d85dcb767a4afcaf7637dfbf1b3., daff3e7fcafd33ff24024100cbc1e9c9=hbase:namespace,,1699781403532.daff3e7fcafd33ff24024100cbc1e9c9., e4f203e272def65afc0882551ad2e3de=ns4:test-16997814143203,,1699781418942.e4f203e272def65afc0882551ad2e3de., 1588230740=hbase:meta,,1.1588230740, 37c7f0bb858c8098fd397daa8ed9cc98=test-1699781414320,,1699781414853.37c7f0bb858c8098fd397daa8ed9cc98., 7dc83426edfa8b37518a7f14a9649f4c=ns2:test-16997814143201,,1699781417119.7dc83426edfa8b37518a7f14a9649f4c., f85d069e361064154880bd8654ab12c0=backup:system_bulk,,1699781423452.f85d069e361064154880bd8654ab12c0., 441fc40990b285ba0c11d5ae2272b2af=backup:system,,1699781421319.441fc40990b285ba0c11d5ae2272b2af.} 2023-11-12T09:31:19,002 DEBUG [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 37c7f0bb858c8098fd397daa8ed9cc98, 441fc40990b285ba0c11d5ae2272b2af, 7dc83426edfa8b37518a7f14a9649f4c, daff3e7fcafd33ff24024100cbc1e9c9, e2e74d85dcb767a4afcaf7637dfbf1b3, e4f203e272def65afc0882551ad2e3de, f85d069e361064154880bd8654ab12c0 2023-11-12T09:31:19,003 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-11-12T09:31:19,003 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-11-12T09:31:19,003 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-11-12T09:31:19,003 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-11-12T09:31:19,003 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-11-12T09:31:19,004 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=13.51 KB heapSize=24 KB 2023-11-12T09:31:19,014 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/ns3/test-16997814143202/e2e74d85dcb767a4afcaf7637dfbf1b3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-11-12T09:31:19,020 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:31:19,021 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns3:test-16997814143202,,1699781417816.e2e74d85dcb767a4afcaf7637dfbf1b3. 2023-11-12T09:31:19,021 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for e2e74d85dcb767a4afcaf7637dfbf1b3: 2023-11-12T09:31:19,021 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-16997814143202,,1699781417816.e2e74d85dcb767a4afcaf7637dfbf1b3. 2023-11-12T09:31:19,023 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing daff3e7fcafd33ff24024100cbc1e9c9, disabling compactions & flushes 2023-11-12T09:31:19,023 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1699781403532.daff3e7fcafd33ff24024100cbc1e9c9. 2023-11-12T09:31:19,023 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1699781403532.daff3e7fcafd33ff24024100cbc1e9c9. 2023-11-12T09:31:19,023 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1699781403532.daff3e7fcafd33ff24024100cbc1e9c9. after waiting 0 ms 2023-11-12T09:31:19,023 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1699781403532.daff3e7fcafd33ff24024100cbc1e9c9. 2023-11-12T09:31:19,023 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing daff3e7fcafd33ff24024100cbc1e9c9 1/1 column families, dataSize=249 B heapSize=1.02 KB 2023-11-12T09:31:19,037 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/hbase/namespace/daff3e7fcafd33ff24024100cbc1e9c9/.tmp/info/365f716759ef4e5b92e5a24fb1db02be is 45, key is default/info:d/1699781404517/Put/seqid=0 2023-11-12T09:31:19,038 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/hbase/meta/1588230740/.tmp/info/e158ce02b78e41ef9594673cbd1bb9b0 is 159, key is ns2:test-16997814143201,,1699781417119.7dc83426edfa8b37518a7f14a9649f4c./info:regioninfo/1699781417512/Put/seqid=0 2023-11-12T09:31:19,202 DEBUG [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 37c7f0bb858c8098fd397daa8ed9cc98, 441fc40990b285ba0c11d5ae2272b2af, 7dc83426edfa8b37518a7f14a9649f4c, daff3e7fcafd33ff24024100cbc1e9c9, e4f203e272def65afc0882551ad2e3de, f85d069e361064154880bd8654ab12c0 2023-11-12T09:31:19,403 DEBUG [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 37c7f0bb858c8098fd397daa8ed9cc98, 441fc40990b285ba0c11d5ae2272b2af, 7dc83426edfa8b37518a7f14a9649f4c, daff3e7fcafd33ff24024100cbc1e9c9, e4f203e272def65afc0882551ad2e3de, f85d069e361064154880bd8654ab12c0 2023-11-12T09:31:19,447 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/hbase/namespace/daff3e7fcafd33ff24024100cbc1e9c9/.tmp/info/365f716759ef4e5b92e5a24fb1db02be 2023-11-12T09:31:19,447 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.12 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/hbase/meta/1588230740/.tmp/info/e158ce02b78e41ef9594673cbd1bb9b0 2023-11-12T09:31:19,454 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/hbase/namespace/daff3e7fcafd33ff24024100cbc1e9c9/.tmp/info/365f716759ef4e5b92e5a24fb1db02be as hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/hbase/namespace/daff3e7fcafd33ff24024100cbc1e9c9/info/365f716759ef4e5b92e5a24fb1db02be 2023-11-12T09:31:19,460 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/hbase/namespace/daff3e7fcafd33ff24024100cbc1e9c9/info/365f716759ef4e5b92e5a24fb1db02be, entries=7, sequenceid=11, filesize=5.1 K 2023-11-12T09:31:19,461 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~249 B/249, heapSize ~1.01 KB/1032, currentSize=0 B/0 for daff3e7fcafd33ff24024100cbc1e9c9 in 438ms, sequenceid=11, compaction requested=false 2023-11-12T09:31:19,461 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-11-12T09:31:19,470 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/hbase/namespace/daff3e7fcafd33ff24024100cbc1e9c9/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2023-11-12T09:31:19,470 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:31:19,472 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1699781403532.daff3e7fcafd33ff24024100cbc1e9c9. 2023-11-12T09:31:19,472 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for daff3e7fcafd33ff24024100cbc1e9c9: 2023-11-12T09:31:19,472 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1699781403532.daff3e7fcafd33ff24024100cbc1e9c9. 2023-11-12T09:31:19,473 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing e4f203e272def65afc0882551ad2e3de, disabling compactions & flushes 2023-11-12T09:31:19,473 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns4:test-16997814143203,,1699781418942.e4f203e272def65afc0882551ad2e3de. 2023-11-12T09:31:19,473 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16997814143203,,1699781418942.e4f203e272def65afc0882551ad2e3de. 2023-11-12T09:31:19,473 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16997814143203,,1699781418942.e4f203e272def65afc0882551ad2e3de. after waiting 0 ms 2023-11-12T09:31:19,473 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16997814143203,,1699781418942.e4f203e272def65afc0882551ad2e3de. 2023-11-12T09:31:19,473 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/hbase/meta/1588230740/.tmp/rep_barrier/e6c71cf2dca645bfad8d25988d6682d9 is 90, key is table1,,1699781433694.5bdf91d1fb4f5b624b93a3ddc48685df./rep_barrier:/1699781439118/DeleteFamily/seqid=0 2023-11-12T09:31:19,476 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/ns4/test-16997814143203/e4f203e272def65afc0882551ad2e3de/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-11-12T09:31:19,477 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:31:19,478 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns4:test-16997814143203,,1699781418942.e4f203e272def65afc0882551ad2e3de. 2023-11-12T09:31:19,478 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for e4f203e272def65afc0882551ad2e3de: 2023-11-12T09:31:19,478 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-16997814143203,,1699781418942.e4f203e272def65afc0882551ad2e3de. 2023-11-12T09:31:19,478 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=172 B at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/hbase/meta/1588230740/.tmp/rep_barrier/e6c71cf2dca645bfad8d25988d6682d9 2023-11-12T09:31:19,478 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 37c7f0bb858c8098fd397daa8ed9cc98, disabling compactions & flushes 2023-11-12T09:31:19,479 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region test-1699781414320,,1699781414853.37c7f0bb858c8098fd397daa8ed9cc98. 2023-11-12T09:31:19,479 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1699781414320,,1699781414853.37c7f0bb858c8098fd397daa8ed9cc98. 2023-11-12T09:31:19,479 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on test-1699781414320,,1699781414853.37c7f0bb858c8098fd397daa8ed9cc98. after waiting 0 ms 2023-11-12T09:31:19,479 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region test-1699781414320,,1699781414853.37c7f0bb858c8098fd397daa8ed9cc98. 2023-11-12T09:31:19,486 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/default/test-1699781414320/37c7f0bb858c8098fd397daa8ed9cc98/recovered.edits/107.seqid, newMaxSeqId=107, maxSeqId=1 2023-11-12T09:31:19,486 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:31:19,487 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed test-1699781414320,,1699781414853.37c7f0bb858c8098fd397daa8ed9cc98. 2023-11-12T09:31:19,487 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 37c7f0bb858c8098fd397daa8ed9cc98: 2023-11-12T09:31:19,488 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1699781414320,,1699781414853.37c7f0bb858c8098fd397daa8ed9cc98. 2023-11-12T09:31:19,489 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 7dc83426edfa8b37518a7f14a9649f4c, disabling compactions & flushes 2023-11-12T09:31:19,489 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns2:test-16997814143201,,1699781417119.7dc83426edfa8b37518a7f14a9649f4c. 2023-11-12T09:31:19,489 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16997814143201,,1699781417119.7dc83426edfa8b37518a7f14a9649f4c. 2023-11-12T09:31:19,489 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16997814143201,,1699781417119.7dc83426edfa8b37518a7f14a9649f4c. after waiting 0 ms 2023-11-12T09:31:19,489 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16997814143201,,1699781417119.7dc83426edfa8b37518a7f14a9649f4c. 2023-11-12T09:31:19,489 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 7dc83426edfa8b37518a7f14a9649f4c 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-11-12T09:31:19,504 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/ns2/test-16997814143201/7dc83426edfa8b37518a7f14a9649f4c/.tmp/f/65e923ed937144b99545aa86ca73282f is 37, key is row10/f:q1/1699781417745/Put/seqid=0 2023-11-12T09:31:19,506 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/hbase/meta/1588230740/.tmp/table/2d42e8226c25493ea693bfc347421d3a is 84, key is table1,,1699781433694.5bdf91d1fb4f5b624b93a3ddc48685df./table:/1699781439118/DeleteFamily/seqid=0 2023-11-12T09:31:19,509 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/ns2/test-16997814143201/7dc83426edfa8b37518a7f14a9649f4c/.tmp/f/65e923ed937144b99545aa86ca73282f 2023-11-12T09:31:19,510 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.22 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/hbase/meta/1588230740/.tmp/table/2d42e8226c25493ea693bfc347421d3a 2023-11-12T09:31:19,516 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/ns2/test-16997814143201/7dc83426edfa8b37518a7f14a9649f4c/.tmp/f/65e923ed937144b99545aa86ca73282f as hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/ns2/test-16997814143201/7dc83426edfa8b37518a7f14a9649f4c/f/65e923ed937144b99545aa86ca73282f 2023-11-12T09:31:19,516 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/hbase/meta/1588230740/.tmp/info/e158ce02b78e41ef9594673cbd1bb9b0 as hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/hbase/meta/1588230740/info/e158ce02b78e41ef9594673cbd1bb9b0 2023-11-12T09:31:19,522 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/ns2/test-16997814143201/7dc83426edfa8b37518a7f14a9649f4c/f/65e923ed937144b99545aa86ca73282f, entries=99, sequenceid=103, filesize=8.2 K 2023-11-12T09:31:19,522 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/hbase/meta/1588230740/info/e158ce02b78e41ef9594673cbd1bb9b0, entries=74, sequenceid=63, filesize=14.4 K 2023-11-12T09:31:19,523 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 7dc83426edfa8b37518a7f14a9649f4c in 34ms, sequenceid=103, compaction requested=false 2023-11-12T09:31:19,530 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/hbase/meta/1588230740/.tmp/rep_barrier/e6c71cf2dca645bfad8d25988d6682d9 as hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/hbase/meta/1588230740/rep_barrier/e6c71cf2dca645bfad8d25988d6682d9 2023-11-12T09:31:19,530 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-11-12T09:31:19,531 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-11-12T09:31:19,535 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/ns2/test-16997814143201/7dc83426edfa8b37518a7f14a9649f4c/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-11-12T09:31:19,536 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:31:19,537 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns2:test-16997814143201,,1699781417119.7dc83426edfa8b37518a7f14a9649f4c. 2023-11-12T09:31:19,537 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 7dc83426edfa8b37518a7f14a9649f4c: 2023-11-12T09:31:19,537 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-16997814143201,,1699781417119.7dc83426edfa8b37518a7f14a9649f4c. 2023-11-12T09:31:19,539 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing f85d069e361064154880bd8654ab12c0, disabling compactions & flushes 2023-11-12T09:31:19,539 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1699781423452.f85d069e361064154880bd8654ab12c0. 2023-11-12T09:31:19,539 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1699781423452.f85d069e361064154880bd8654ab12c0. 2023-11-12T09:31:19,539 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1699781423452.f85d069e361064154880bd8654ab12c0. after waiting 0 ms 2023-11-12T09:31:19,539 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1699781423452.f85d069e361064154880bd8654ab12c0. 2023-11-12T09:31:19,540 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/hbase/meta/1588230740/rep_barrier/e6c71cf2dca645bfad8d25988d6682d9, entries=2, sequenceid=63, filesize=5.4 K 2023-11-12T09:31:19,540 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/hbase/meta/1588230740/.tmp/table/2d42e8226c25493ea693bfc347421d3a as hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/hbase/meta/1588230740/table/2d42e8226c25493ea693bfc347421d3a 2023-11-12T09:31:19,542 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system_bulk/f85d069e361064154880bd8654ab12c0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-11-12T09:31:19,543 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:31:19,544 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system_bulk,,1699781423452.f85d069e361064154880bd8654ab12c0. 2023-11-12T09:31:19,544 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for f85d069e361064154880bd8654ab12c0: 2023-11-12T09:31:19,544 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1699781423452.f85d069e361064154880bd8654ab12c0. 2023-11-12T09:31:19,545 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 441fc40990b285ba0c11d5ae2272b2af, disabling compactions & flushes 2023-11-12T09:31:19,545 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system,,1699781421319.441fc40990b285ba0c11d5ae2272b2af. 2023-11-12T09:31:19,545 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1699781421319.441fc40990b285ba0c11d5ae2272b2af. 2023-11-12T09:31:19,545 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1699781421319.441fc40990b285ba0c11d5ae2272b2af. after waiting 0 ms 2023-11-12T09:31:19,545 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1699781421319.441fc40990b285ba0c11d5ae2272b2af. 2023-11-12T09:31:19,545 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 441fc40990b285ba0c11d5ae2272b2af 2/2 column families, dataSize=985 B heapSize=1.97 KB 2023-11-12T09:31:19,548 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/hbase/meta/1588230740/table/2d42e8226c25493ea693bfc347421d3a, entries=17, sequenceid=63, filesize=6.0 K 2023-11-12T09:31:19,548 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/.tmp/meta/941da6c733064f36bde9bcfd39762446 is 169, key is trslm:hdfs://localhost:41663/backupUT\x00test-1699781414320/meta:log-roll-map/1699781448373/Put/seqid=0 2023-11-12T09:31:19,549 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~13.51 KB/13834, heapSize ~23.95 KB/24528, currentSize=0 B/0 for 1588230740 in 546ms, sequenceid=63, compaction requested=false 2023-11-12T09:31:19,549 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-11-12T09:31:19,570 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/hbase/meta/1588230740/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=1 2023-11-12T09:31:19,571 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:31:19,571 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-11-12T09:31:19,572 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-11-12T09:31:19,572 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-11-12T09:31:19,572 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-11-12T09:31:19,603 DEBUG [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(1609): Waiting on 441fc40990b285ba0c11d5ae2272b2af 2023-11-12T09:31:19,803 DEBUG [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(1609): Waiting on 441fc40990b285ba0c11d5ae2272b2af 2023-11-12T09:31:19,961 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=27 (bloomFilter=true), to=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/.tmp/meta/941da6c733064f36bde9bcfd39762446 2023-11-12T09:31:19,969 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/.tmp/session/4da304998c334925b79f11053776f3f6 is 310, key is session:backup_1699781439252/session:context/1699781448798/Put/seqid=0 2023-11-12T09:31:20,003 INFO [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(1579): Waiting on 1 regions to close 2023-11-12T09:31:20,004 DEBUG [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(1583): Online Regions={441fc40990b285ba0c11d5ae2272b2af=backup:system,,1699781421319.441fc40990b285ba0c11d5ae2272b2af.} 2023-11-12T09:31:20,004 DEBUG [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(1609): Waiting on 441fc40990b285ba0c11d5ae2272b2af 2023-11-12T09:31:20,204 DEBUG [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(1609): Waiting on 441fc40990b285ba0c11d5ae2272b2af 2023-11-12T09:31:20,373 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=534 B at sequenceid=27 (bloomFilter=true), to=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/.tmp/session/4da304998c334925b79f11053776f3f6 2023-11-12T09:31:20,389 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/.tmp/meta/941da6c733064f36bde9bcfd39762446 as hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/meta/941da6c733064f36bde9bcfd39762446 2023-11-12T09:31:20,396 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/meta/941da6c733064f36bde9bcfd39762446, entries=4, sequenceid=27, filesize=5.5 K 2023-11-12T09:31:20,397 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/.tmp/session/4da304998c334925b79f11053776f3f6 as hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/session/4da304998c334925b79f11053776f3f6 2023-11-12T09:31:20,402 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/session/4da304998c334925b79f11053776f3f6, entries=2, sequenceid=27, filesize=5.3 K 2023-11-12T09:31:20,403 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~985 B/985, heapSize ~1.94 KB/1984, currentSize=0 B/0 for 441fc40990b285ba0c11d5ae2272b2af in 858ms, sequenceid=27, compaction requested=true 2023-11-12T09:31:20,409 DEBUG [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(1609): Waiting on 441fc40990b285ba0c11d5ae2272b2af 2023-11-12T09:31:20,425 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/data/backup/system/441fc40990b285ba0c11d5ae2272b2af/recovered.edits/30.seqid, newMaxSeqId=30, maxSeqId=1 2023-11-12T09:31:20,426 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:31:20,428 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system,,1699781421319.441fc40990b285ba0c11d5ae2272b2af. 2023-11-12T09:31:20,428 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 441fc40990b285ba0c11d5ae2272b2af: 2023-11-12T09:31:20,428 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1699781421319.441fc40990b285ba0c11d5ae2272b2af. 2023-11-12T09:31:20,479 INFO [Socket Reader #1 for port 0 {}] ipc.Server$Connection(2001): Auth successful for appattempt_1699781452016_0001_000001 (auth:SIMPLE) 2023-11-12T09:31:20,610 INFO [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,37573,1699781400943; all regions closed. 2023-11-12T09:31:20,619 DEBUG [RS:0;jenkins-hbase3:37573 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/oldWALs 2023-11-12T09:31:20,619 INFO [RS:0;jenkins-hbase3:37573 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C37573%2C1699781400943.meta:.meta(num 1699781443428) 2023-11-12T09:31:20,626 DEBUG [RS:0;jenkins-hbase3:37573 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/oldWALs 2023-11-12T09:31:20,626 INFO [RS:0;jenkins-hbase3:37573 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C37573%2C1699781400943:(num 1699781443446) 2023-11-12T09:31:20,627 DEBUG [RS:0;jenkins-hbase3:37573 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:31:20,627 INFO [RS:0;jenkins-hbase3:37573 {}] regionserver.LeaseManager(133): Closed leases 2023-11-12T09:31:20,627 INFO [RS:0;jenkins-hbase3:37573 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2023-11-12T09:31:20,627 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-11-12T09:31:20,629 INFO [RS:0;jenkins-hbase3:37573 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:37573 2023-11-12T09:31:20,632 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-11-12T09:31:20,632 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rs/jenkins-hbase3.apache.org,37573,1699781400943 2023-11-12T09:31:20,634 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,37573,1699781400943] 2023-11-12T09:31:20,634 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,37573,1699781400943; numProcessing=1 2023-11-12T09:31:20,636 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(224): Node /1/draining/jenkins-hbase3.apache.org,37573,1699781400943 already deleted, retry=false 2023-11-12T09:31:20,636 INFO [RegionServerTracker-0 {}] master.ServerManager(561): Cluster shutdown set; jenkins-hbase3.apache.org,37573,1699781400943 expired; onlineServers=0 2023-11-12T09:31:20,636 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,44289,1699781397811' ***** 2023-11-12T09:31:20,636 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2531): STOPPED: Cluster shutdown set; onlineServer=0 2023-11-12T09:31:20,637 DEBUG [M:0;jenkins-hbase3:44289 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ace50c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-11-12T09:31:20,637 INFO [M:0;jenkins-hbase3:44289 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-11-12T09:31:20,655 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/master 2023-11-12T09:31:20,655 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:31:20,665 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-11-12T09:31:20,672 INFO [M:0;jenkins-hbase3:44289 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@613d33f5{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-11-12T09:31:20,672 INFO [M:0;jenkins-hbase3:44289 {}] server.AbstractConnector(383): Stopped ServerConnector@17469d4d{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-11-12T09:31:20,672 INFO [M:0;jenkins-hbase3:44289 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-11-12T09:31:20,674 INFO [M:0;jenkins-hbase3:44289 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@8701aa9{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-11-12T09:31:20,675 INFO [M:0;jenkins-hbase3:44289 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@2505f075{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/hadoop.log.dir/,STOPPED} 2023-11-12T09:31:20,675 INFO [M:0;jenkins-hbase3:44289 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,44289,1699781397811 2023-11-12T09:31:20,675 INFO [M:0;jenkins-hbase3:44289 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,44289,1699781397811; all regions closed. 2023-11-12T09:31:20,675 DEBUG [M:0;jenkins-hbase3:44289 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:31:20,675 INFO [M:0;jenkins-hbase3:44289 {}] master.HMaster(1666): Stopping master jetty server 2023-11-12T09:31:20,676 INFO [M:0;jenkins-hbase3:44289 {}] server.AbstractConnector(383): Stopped ServerConnector@31bf9364{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-11-12T09:31:20,676 DEBUG [M:0;jenkins-hbase3:44289 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-11-12T09:31:20,676 INFO [M:0;jenkins-hbase3:44289 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-11-12T09:31:20,676 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-11-12T09:31:20,676 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1699781402357 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1699781402357,5,FailOnTimeoutGroup] 2023-11-12T09:31:20,676 DEBUG [M:0;jenkins-hbase3:44289 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x50f72ab6 to 127.0.0.1:57951 2023-11-12T09:31:20,676 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1699781402358 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1699781402358,5,FailOnTimeoutGroup] 2023-11-12T09:31:20,676 DEBUG [M:0;jenkins-hbase3:44289 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:31:20,676 DEBUG [M:0;jenkins-hbase3:44289 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-11-12T09:31:20,677 INFO [M:0;jenkins-hbase3:44289 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-11-12T09:31:20,677 DEBUG [M:0;jenkins-hbase3:44289 {}] master.HMaster(1689): Stopping service threads 2023-11-12T09:31:20,677 INFO [M:0;jenkins-hbase3:44289 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-11-12T09:31:20,677 ERROR [M:0;jenkins-hbase3:44289 {}] procedure2.ProcedureExecutor(654): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-2,5,PEWorkerGroup] 2023-11-12T09:31:20,677 INFO [M:0;jenkins-hbase3:44289 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-11-12T09:31:20,677 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-11-12T09:31:20,677 DEBUG [M:0;jenkins-hbase3:44289 {}] zookeeper.ZKUtil(347): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Unable to get data of znode /1/master because node does not exist (not an error) 2023-11-12T09:31:20,678 WARN [M:0;jenkins-hbase3:44289 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-11-12T09:31:20,678 INFO [M:0;jenkins-hbase3:44289 {}] assignment.AssignmentManager(383): Stopping assignment manager 2023-11-12T09:31:20,678 INFO [M:0;jenkins-hbase3:44289 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-11-12T09:31:20,679 DEBUG [M:0;jenkins-hbase3:44289 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-11-12T09:31:20,690 INFO [M:0;jenkins-hbase3:44289 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:31:20,690 DEBUG [M:0;jenkins-hbase3:44289 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:31:20,690 DEBUG [M:0;jenkins-hbase3:44289 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-11-12T09:31:20,690 DEBUG [M:0;jenkins-hbase3:44289 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:31:20,690 INFO [M:0;jenkins-hbase3:44289 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=203.17 KB heapSize=244.13 KB 2023-11-12T09:31:20,711 DEBUG [M:0;jenkins-hbase3:44289 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/05c3d4516e464bee84484dead298912a is 88, key is hbase:meta,,1/info:sn/1699781402832/Put/seqid=0 2023-11-12T09:31:20,734 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-11-12T09:31:20,735 INFO [RS:0;jenkins-hbase3:37573 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,37573,1699781400943; zookeeper connection closed. 2023-11-12T09:31:20,735 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37573-0x1004cf7b7840001, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-11-12T09:31:20,735 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@311c5cff {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@311c5cff 2023-11-12T09:31:20,735 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-11-12T09:31:21,116 INFO [M:0;jenkins-hbase3:44289 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/05c3d4516e464bee84484dead298912a 2023-11-12T09:31:21,136 DEBUG [M:0;jenkins-hbase3:44289 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1f0fbfad5569450b830fd56b4229715d is 988, key is \x00\x00\x00\x00\x00\x00\x00\x1D/proc:d/1699781424654/Put/seqid=0 2023-11-12T09:31:21,540 INFO [M:0;jenkins-hbase3:44289 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=202.57 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1f0fbfad5569450b830fd56b4229715d 2023-11-12T09:31:21,545 INFO [M:0;jenkins-hbase3:44289 {}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1f0fbfad5569450b830fd56b4229715d 2023-11-12T09:31:21,558 DEBUG [M:0;jenkins-hbase3:44289 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ef23af599eaa4323aa70ff83524ef9e0 is 82, key is jenkins-hbase3.apache.org,37573,1699781400943/rs:state/1699781402429/Put/seqid=0 2023-11-12T09:31:21,572 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:31:21,962 INFO [M:0;jenkins-hbase3:44289 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ef23af599eaa4323aa70ff83524ef9e0 2023-11-12T09:31:21,968 DEBUG [M:0;jenkins-hbase3:44289 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/05c3d4516e464bee84484dead298912a as hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/05c3d4516e464bee84484dead298912a 2023-11-12T09:31:21,973 INFO [M:0;jenkins-hbase3:44289 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/05c3d4516e464bee84484dead298912a, entries=8, sequenceid=480, filesize=5.5 K 2023-11-12T09:31:21,974 DEBUG [M:0;jenkins-hbase3:44289 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1f0fbfad5569450b830fd56b4229715d as hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1f0fbfad5569450b830fd56b4229715d 2023-11-12T09:31:21,978 INFO [M:0;jenkins-hbase3:44289 {}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1f0fbfad5569450b830fd56b4229715d 2023-11-12T09:31:21,978 INFO [M:0;jenkins-hbase3:44289 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1f0fbfad5569450b830fd56b4229715d, entries=53, sequenceid=480, filesize=16.7 K 2023-11-12T09:31:21,979 DEBUG [M:0;jenkins-hbase3:44289 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ef23af599eaa4323aa70ff83524ef9e0 as hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ef23af599eaa4323aa70ff83524ef9e0 2023-11-12T09:31:21,983 INFO [M:0;jenkins-hbase3:44289 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38625/user/jenkins/test-data/53742d32-c2f5-f789-3d94-9f3463d2e73a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ef23af599eaa4323aa70ff83524ef9e0, entries=1, sequenceid=480, filesize=5.1 K 2023-11-12T09:31:21,984 INFO [M:0;jenkins-hbase3:44289 {}] regionserver.HRegion(3022): Finished flush of dataSize ~203.17 KB/208043, heapSize ~243.84 KB/249688, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1294ms, sequenceid=480, compaction requested=false 2023-11-12T09:31:21,993 INFO [M:0;jenkins-hbase3:44289 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:31:21,993 DEBUG [M:0;jenkins-hbase3:44289 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-11-12T09:31:21,996 INFO [M:0;jenkins-hbase3:44289 {}] flush.MasterFlushTableProcedureManager(85): stop: server shutting down. 2023-11-12T09:31:21,996 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-11-12T09:31:21,996 INFO [M:0;jenkins-hbase3:44289 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-11-12T09:31:21,998 INFO [M:0;jenkins-hbase3:44289 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:44289 2023-11-12T09:31:22,000 DEBUG [M:0;jenkins-hbase3:44289 {}] zookeeper.RecoverableZooKeeper(224): Node /1/rs/jenkins-hbase3.apache.org,44289,1699781397811 already deleted, retry=false 2023-11-12T09:31:22,102 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-11-12T09:31:22,102 INFO [M:0;jenkins-hbase3:44289 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,44289,1699781397811; zookeeper connection closed. 2023-11-12T09:31:22,102 DEBUG [Listener at localhost/33247-EventThread {}] zookeeper.ZKWatcher(604): master:44289-0x1004cf7b7840000, quorum=127.0.0.1:57951, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-11-12T09:31:22,106 WARN [Listener at jenkins-hbase3.apache.org/39019 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-11-12T09:31:22,111 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-11-12T09:31:22,219 WARN [BP-416291311-172.31.12.81-1699781394581 heartbeating to localhost/127.0.0.1:38625 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-11-12T09:31:22,219 WARN [BP-416291311-172.31.12.81-1699781394581 heartbeating to localhost/127.0.0.1:38625 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-416291311-172.31.12.81-1699781394581 (Datanode Uuid 188c7465-4427-4b07-a6c4-b29194f7812c) service to localhost/127.0.0.1:38625 2023-11-12T09:31:22,220 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/cluster_1b67588a-7841-a8ad-b22d-cb19219656d1/dfs/data/data1/current/BP-416291311-172.31.12.81-1699781394581 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-11-12T09:31:22,220 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/80f2cbd9-a5fb-077b-0cb5-86f2a8672e27/cluster_1b67588a-7841-a8ad-b22d-cb19219656d1/dfs/data/data2/current/BP-416291311-172.31.12.81-1699781394581 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-11-12T09:31:22,229 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-11-12T09:31:22,347 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2023-11-12T09:31:22,382 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-11-12T09:31:22,382 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] hbase.HBaseTestingUtility(2857): Stopping mini mapreduce cluster... 2023-11-12T09:31:22,395 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-11-12T09:31:22,518 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-11-12T09:31:22,633 ERROR [Thread[Thread-492,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-11-12T09:31:22,633 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-11-12T09:31:22,740 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2023-11-12T09:31:22,750 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(61): Returning, interrupted : java.lang.InterruptedException 2023-11-12T09:31:22,751 ERROR [Thread[Thread-502,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-11-12T09:31:22,760 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-11-12T09:31:22,865 ERROR [Thread[Thread-467,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-11-12T09:31:22,865 INFO [Listener at jenkins-hbase3.apache.org/39019 {}] hbase.HBaseTestingUtility(2860): Mini mapreduce cluster stopped