2023-09-06 20:34:56,809 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@13c9d689 2023-09-06 20:34:56,821 main DEBUG Took 0.010327 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-09-06 20:34:56,822 main DEBUG PluginManager 'Core' found 129 plugins 2023-09-06 20:34:56,822 main DEBUG PluginManager 'Level' found 0 plugins 2023-09-06 20:34:56,823 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-09-06 20:34:56,824 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:34:56,832 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-09-06 20:34:56,843 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:34:56,845 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:34:56,845 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:34:56,846 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:34:56,846 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:34:56,846 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:34:56,847 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:34:56,847 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:34:56,847 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:34:56,848 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:34:56,848 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:34:56,849 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:34:56,849 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:34:56,849 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:34:56,850 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:34:56,850 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:34:56,850 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:34:56,850 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:34:56,851 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:34:56,851 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:34:56,851 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:34:56,851 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:34:56,852 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:34:56,852 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:34:56,852 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:34:56,853 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-09-06 20:34:56,855 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:34:56,856 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-09-06 20:34:56,858 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-09-06 20:34:56,858 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-09-06 20:34:56,859 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-09-06 20:34:56,860 main DEBUG PluginManager 'Converter' found 47 plugins 2023-09-06 20:34:56,871 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-09-06 20:34:56,874 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-09-06 20:34:56,878 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-09-06 20:34:56,878 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-09-06 20:34:56,878 main DEBUG createAppenders(={Console}) 2023-09-06 20:34:56,879 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@13c9d689 initialized 2023-09-06 20:34:56,879 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@13c9d689 2023-09-06 20:34:56,880 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@13c9d689 OK. 2023-09-06 20:34:56,880 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-09-06 20:34:56,881 main DEBUG OutputStream closed 2023-09-06 20:34:56,881 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-09-06 20:34:56,881 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-09-06 20:34:56,881 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@15c43bd9 OK 2023-09-06 20:34:56,934 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-09-06 20:34:56,936 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-09-06 20:34:56,937 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-09-06 20:34:56,938 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-09-06 20:34:56,939 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-09-06 20:34:56,939 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-09-06 20:34:56,939 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-09-06 20:34:56,940 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-09-06 20:34:56,940 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-09-06 20:34:56,940 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-09-06 20:34:56,940 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-09-06 20:34:56,941 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-09-06 20:34:56,941 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-09-06 20:34:56,941 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-09-06 20:34:56,941 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-09-06 20:34:56,942 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-09-06 20:34:56,942 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-09-06 20:34:56,943 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-09-06 20:34:56,946 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-09-06 20:34:56,946 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@1b6e1eff) with optional ClassLoader: null 2023-09-06 20:34:56,946 main DEBUG Shutdown hook enabled. Registering a new one. 2023-09-06 20:34:56,947 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@1b6e1eff] started OK. 2023-09-06T20:34:56,980 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestRemoteRestore timeout: 13 mins 2023-09-06 20:34:56,984 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-09-06 20:34:56,984 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-09-06T20:34:57,321 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930 2023-09-06T20:34:57,326 DEBUG [Time-limited test {}] impl.BackupManager(127): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-09-06T20:34:57,329 DEBUG [Time-limited test {}] impl.BackupManager(157): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:34:57,330 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-09-06T20:34:57,333 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/cluster_8944b83c-4af0-4d3a-395e-af5ae00b91a1, deleteOnExit=true 2023-09-06T20:34:57,333 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-09-06T20:34:57,334 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/test.cache.data in system properties and HBase conf 2023-09-06T20:34:57,334 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/hadoop.tmp.dir in system properties and HBase conf 2023-09-06T20:34:57,335 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/hadoop.log.dir in system properties and HBase conf 2023-09-06T20:34:57,335 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/mapreduce.cluster.local.dir in system properties and HBase conf 2023-09-06T20:34:57,335 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-09-06T20:34:57,336 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-09-06T20:34:57,454 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-09-06T20:34:57,881 DEBUG [Time-limited test {}] fs.HFileSystem(308): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-09-06T20:34:57,887 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-09-06T20:34:57,887 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-09-06T20:34:57,888 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-09-06T20:34:57,888 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-09-06T20:34:57,888 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-09-06T20:34:57,889 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-09-06T20:34:57,889 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-09-06T20:34:57,890 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/dfs.journalnode.edits.dir in system properties and HBase conf 2023-09-06T20:34:57,890 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-09-06T20:34:57,890 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/nfs.dump.dir in system properties and HBase conf 2023-09-06T20:34:57,891 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/java.io.tmpdir in system properties and HBase conf 2023-09-06T20:34:57,891 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/dfs.journalnode.edits.dir in system properties and HBase conf 2023-09-06T20:34:57,892 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-09-06T20:34:57,892 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-09-06T20:34:58,325 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-09-06T20:34:58,329 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-09-06T20:34:58,621 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-09-06T20:34:58,773 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@7b7a5551 via org.mortbay.log.Slf4jLog 2023-09-06T20:34:58,789 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:34:58,828 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:34:58,865 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/java.io.tmpdir/Jetty_localhost_44899_hdfs____eaagle/webapp 2023-09-06T20:34:58,998 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:44899 2023-09-06T20:34:59,025 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-09-06T20:34:59,025 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-09-06T20:34:59,576 WARN [Listener at localhost/36869 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-09-06T20:34:59,639 WARN [Listener at localhost/36869 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-09-06T20:34:59,656 WARN [Listener at localhost/36869 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:34:59,664 INFO [Listener at localhost/36869 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:34:59,669 INFO [Listener at localhost/36869 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/java.io.tmpdir/Jetty_localhost_40395_datanode____.bkoy6e/webapp 2023-09-06T20:34:59,767 INFO [Listener at localhost/36869 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:40395 2023-09-06T20:35:00,053 WARN [Listener at localhost/43069 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-09-06T20:35:00,327 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-09-06T20:35:00,491 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x2c4e90dc9dc9a799: Processing first storage report for DS-1307a321-484a-4087-acdc-931715071dc3 from datanode 8043810c-8db8-439d-b251-34c1f12b8a82 2023-09-06T20:35:00,492 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x2c4e90dc9dc9a799: from storage DS-1307a321-484a-4087-acdc-931715071dc3 node DatanodeRegistration(127.0.0.1:46843, datanodeUuid=8043810c-8db8-439d-b251-34c1f12b8a82, infoPort=44309, infoSecurePort=0, ipcPort=43069, storageInfo=lv=-57;cid=testClusterID;nsid=921173290;c=1694032498405), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2023-09-06T20:35:00,493 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x2c4e90dc9dc9a799: Processing first storage report for DS-fdeac9fe-a230-4498-b811-3ee997a5f5dc from datanode 8043810c-8db8-439d-b251-34c1f12b8a82 2023-09-06T20:35:00,493 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x2c4e90dc9dc9a799: from storage DS-fdeac9fe-a230-4498-b811-3ee997a5f5dc node DatanodeRegistration(127.0.0.1:46843, datanodeUuid=8043810c-8db8-439d-b251-34c1f12b8a82, infoPort=44309, infoSecurePort=0, ipcPort=43069, storageInfo=lv=-57;cid=testClusterID;nsid=921173290;c=1694032498405), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-09-06T20:35:00,570 DEBUG [Listener at localhost/43069 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930 2023-09-06T20:35:00,655 INFO [Listener at localhost/43069 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/cluster_8944b83c-4af0-4d3a-395e-af5ae00b91a1/zookeeper_0, clientPort=50423, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/cluster_8944b83c-4af0-4d3a-395e-af5ae00b91a1/zookeeper_0/version-2, dataDirSize=424 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/cluster_8944b83c-4af0-4d3a-395e-af5ae00b91a1/zookeeper_0/version-2, dataLogSize=424 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, serverId=0 2023-09-06T20:35:00,674 INFO [Listener at localhost/43069 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=50423 2023-09-06T20:35:00,682 INFO [Listener at localhost/43069 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:35:00,684 INFO [Listener at localhost/43069 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:35:01,352 INFO [Listener at localhost/43069 {}] util.FSUtils(461): Created version file at hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd with version=8 2023-09-06T20:35:01,353 INFO [Listener at localhost/43069 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/hbase-staging 2023-09-06T20:35:01,369 DEBUG [Listener at localhost/43069 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-09-06T20:35:01,369 DEBUG [Listener at localhost/43069 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-09-06T20:35:01,369 DEBUG [Listener at localhost/43069 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-09-06T20:35:01,369 DEBUG [Listener at localhost/43069 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-09-06T20:35:01,586 DEBUG [Listener at localhost/43069 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-09-06T20:35:02,326 INFO [Listener at localhost/43069 {}] client.ConnectionUtils(128): master/jenkins-hbase3:0 server-side Connection retries=45 2023-09-06T20:35:02,372 INFO [Listener at localhost/43069 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-06T20:35:02,372 INFO [Listener at localhost/43069 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-09-06T20:35:02,372 INFO [Listener at localhost/43069 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-09-06T20:35:02,373 INFO [Listener at localhost/43069 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-06T20:35:02,373 INFO [Listener at localhost/43069 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-09-06T20:35:02,522 INFO [Listener at localhost/43069 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-09-06T20:35:02,649 INFO [Listener at localhost/43069 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-09-06T20:35:02,667 DEBUG [Listener at localhost/43069 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-09-06T20:35:02,674 INFO [Listener at localhost/43069 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-09-06T20:35:02,723 DEBUG [Listener at localhost/43069 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 3415 (auto-detected) 2023-09-06T20:35:02,725 DEBUG [Listener at localhost/43069 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:1c:ff:fe:e2:13:ea (auto-detected) 2023-09-06T20:35:02,762 INFO [Listener at localhost/43069 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:33223 2023-09-06T20:35:02,776 INFO [Listener at localhost/43069 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:35:02,778 INFO [Listener at localhost/43069 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:35:02,799 INFO [Listener at localhost/43069 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:33223 connecting to ZooKeeper ensemble=127.0.0.1:50423 2023-09-06T20:35:02,846 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:332230x0, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-09-06T20:35:02,850 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:33223-0x100eefdf7f60000 connected 2023-09-06T20:35:02,879 DEBUG [Listener at localhost/43069 {}] zookeeper.ZKUtil(113): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-09-06T20:35:02,882 DEBUG [Listener at localhost/43069 {}] zookeeper.ZKUtil(113): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-09-06T20:35:02,886 DEBUG [Listener at localhost/43069 {}] zookeeper.ZKUtil(113): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-09-06T20:35:02,896 DEBUG [Listener at localhost/43069 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33223 2023-09-06T20:35:02,896 DEBUG [Listener at localhost/43069 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33223 2023-09-06T20:35:02,897 DEBUG [Listener at localhost/43069 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33223 2023-09-06T20:35:02,897 DEBUG [Listener at localhost/43069 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33223 2023-09-06T20:35:02,898 DEBUG [Listener at localhost/43069 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33223 2023-09-06T20:35:02,951 INFO [Listener at localhost/43069 {}] log.Log(170): Logging initialized @6790ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2023-09-06T20:35:03,076 INFO [Listener at localhost/43069 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-09-06T20:35:03,076 INFO [Listener at localhost/43069 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-09-06T20:35:03,077 INFO [Listener at localhost/43069 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-09-06T20:35:03,079 INFO [Listener at localhost/43069 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-09-06T20:35:03,080 INFO [Listener at localhost/43069 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-09-06T20:35:03,080 INFO [Listener at localhost/43069 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-09-06T20:35:03,083 INFO [Listener at localhost/43069 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-09-06T20:35:03,211 INFO [Listener at localhost/43069 {}] http.HttpServer(1219): Jetty bound to port 38881 2023-09-06T20:35:03,213 INFO [Listener at localhost/43069 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-09-06T20:35:03,243 INFO [Listener at localhost/43069 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:35:03,247 INFO [Listener at localhost/43069 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@24da0689{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/hadoop.log.dir/,AVAILABLE} 2023-09-06T20:35:03,247 INFO [Listener at localhost/43069 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:35:03,248 INFO [Listener at localhost/43069 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@4c63d339{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-09-06T20:35:03,316 INFO [Listener at localhost/43069 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-09-06T20:35:03,329 INFO [Listener at localhost/43069 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-09-06T20:35:03,330 INFO [Listener at localhost/43069 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-09-06T20:35:03,332 INFO [Listener at localhost/43069 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-09-06T20:35:03,339 INFO [Listener at localhost/43069 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:35:04,482 INFO [Listener at localhost/43069 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@25a564df{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-09-06T20:35:04,494 INFO [Listener at localhost/43069 {}] server.AbstractConnector(333): Started ServerConnector@4420f3e7{HTTP/1.1, (http/1.1)}{0.0.0.0:38881} 2023-09-06T20:35:04,494 INFO [Listener at localhost/43069 {}] server.Server(415): Started @8333ms 2023-09-06T20:35:04,498 INFO [Listener at localhost/43069 {}] master.HMaster(485): hbase.rootdir=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd, hbase.cluster.distributed=false 2023-09-06T20:35:04,562 INFO [Listener at localhost/43069 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase3:0 server-side Connection retries=45 2023-09-06T20:35:04,563 INFO [Listener at localhost/43069 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-06T20:35:04,563 INFO [Listener at localhost/43069 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-09-06T20:35:04,563 INFO [Listener at localhost/43069 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-09-06T20:35:04,563 INFO [Listener at localhost/43069 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-06T20:35:04,563 INFO [Listener at localhost/43069 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-09-06T20:35:04,571 INFO [Listener at localhost/43069 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-09-06T20:35:04,574 INFO [Listener at localhost/43069 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-09-06T20:35:04,575 INFO [Listener at localhost/43069 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:36075 2023-09-06T20:35:04,578 INFO [Listener at localhost/43069 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-09-06T20:35:04,586 DEBUG [Listener at localhost/43069 {}] mob.MobFileCache(120): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-09-06T20:35:04,587 INFO [Listener at localhost/43069 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:35:04,591 INFO [Listener at localhost/43069 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:35:04,594 INFO [Listener at localhost/43069 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:36075 connecting to ZooKeeper ensemble=127.0.0.1:50423 2023-09-06T20:35:04,597 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): regionserver:360750x0, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-09-06T20:35:04,598 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:36075-0x100eefdf7f60001 connected 2023-09-06T20:35:04,598 DEBUG [Listener at localhost/43069 {}] zookeeper.ZKUtil(113): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-09-06T20:35:04,600 DEBUG [Listener at localhost/43069 {}] zookeeper.ZKUtil(113): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-09-06T20:35:04,600 DEBUG [Listener at localhost/43069 {}] zookeeper.ZKUtil(113): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-09-06T20:35:04,601 DEBUG [Listener at localhost/43069 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36075 2023-09-06T20:35:04,602 DEBUG [Listener at localhost/43069 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36075 2023-09-06T20:35:04,602 DEBUG [Listener at localhost/43069 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36075 2023-09-06T20:35:04,603 DEBUG [Listener at localhost/43069 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36075 2023-09-06T20:35:04,603 DEBUG [Listener at localhost/43069 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36075 2023-09-06T20:35:04,606 INFO [Listener at localhost/43069 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-09-06T20:35:04,606 INFO [Listener at localhost/43069 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-09-06T20:35:04,606 INFO [Listener at localhost/43069 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-09-06T20:35:04,607 INFO [Listener at localhost/43069 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-09-06T20:35:04,607 INFO [Listener at localhost/43069 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-09-06T20:35:04,607 INFO [Listener at localhost/43069 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-09-06T20:35:04,608 INFO [Listener at localhost/43069 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-09-06T20:35:04,610 INFO [Listener at localhost/43069 {}] http.HttpServer(1219): Jetty bound to port 37517 2023-09-06T20:35:04,610 INFO [Listener at localhost/43069 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-09-06T20:35:04,611 INFO [Listener at localhost/43069 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:35:04,612 INFO [Listener at localhost/43069 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@5023341c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/hadoop.log.dir/,AVAILABLE} 2023-09-06T20:35:04,612 INFO [Listener at localhost/43069 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:35:04,612 INFO [Listener at localhost/43069 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@2d65fb0f{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-09-06T20:35:04,624 INFO [Listener at localhost/43069 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-09-06T20:35:04,625 INFO [Listener at localhost/43069 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-09-06T20:35:04,625 INFO [Listener at localhost/43069 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-09-06T20:35:04,626 INFO [Listener at localhost/43069 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-09-06T20:35:04,627 INFO [Listener at localhost/43069 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:35:04,631 INFO [Listener at localhost/43069 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@48191814{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-09-06T20:35:04,633 INFO [Listener at localhost/43069 {}] server.AbstractConnector(333): Started ServerConnector@98b5321{HTTP/1.1, (http/1.1)}{0.0.0.0:37517} 2023-09-06T20:35:04,633 INFO [Listener at localhost/43069 {}] server.Server(415): Started @8471ms 2023-09-06T20:35:04,638 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-09-06T20:35:04,641 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@68792fe9{HTTP/1.1, (http/1.1)}{0.0.0.0:40523} 2023-09-06T20:35:04,641 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(415): Started @8479ms 2023-09-06T20:35:04,641 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2388): Adding backup master ZNode /1/backup-masters/jenkins-hbase3.apache.org,33223,1694032501568 2023-09-06T20:35:04,657 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-09-06T20:35:04,657 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-09-06T20:35:04,659 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/jenkins-hbase3.apache.org,33223,1694032501568 2023-09-06T20:35:04,662 DEBUG [M:0;jenkins-hbase3:33223 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:33223 2023-09-06T20:35:04,678 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-09-06T20:35:04,678 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-09-06T20:35:04,678 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:35:04,678 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:35:04,680 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Set watcher on existing znode=/1/master 2023-09-06T20:35:04,681 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Set watcher on existing znode=/1/master 2023-09-06T20:35:04,681 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/jenkins-hbase3.apache.org,33223,1694032501568 from backup master directory 2023-09-06T20:35:04,685 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/jenkins-hbase3.apache.org,33223,1694032501568 2023-09-06T20:35:04,685 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-09-06T20:35:04,686 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-09-06T20:35:04,686 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-09-06T20:35:04,687 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,33223,1694032501568 2023-09-06T20:35:04,690 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-09-06T20:35:04,692 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-09-06T20:35:04,774 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(610): Created cluster ID file at hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/hbase.id with ID: dfcf656f-3e2c-409d-a4c9-cc1fca9f9955 2023-09-06T20:35:04,819 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:35:04,835 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:35:04,835 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:35:04,879 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:35:04,881 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-09-06T20:35:04,901 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(270): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-09-06T20:35:04,901 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(276): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-09-06T20:35:04,903 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(285): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:283) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:310) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:951) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2413) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:574) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:571) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:35:04,941 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:951) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2413) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:574) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:571) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:35:04,943 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-09-06T20:35:05,015 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/MasterData/data/master/store 2023-09-06T20:35:05,439 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-09-06T20:35:05,440 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:05,440 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-09-06T20:35:05,440 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:35:05,441 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:35:05,441 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-09-06T20:35:05,441 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:35:05,441 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:35:05,441 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-09-06T20:35:05,443 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/MasterData/data/master/store/.initializing 2023-09-06T20:35:05,444 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/MasterData/WALs/jenkins-hbase3.apache.org,33223,1694032501568 2023-09-06T20:35:05,453 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-09-06T20:35:05,468 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C33223%2C1694032501568, suffix=, logDir=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/MasterData/WALs/jenkins-hbase3.apache.org,33223,1694032501568, archiveDir=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/MasterData/oldWALs, maxLogs=10 2023-09-06T20:35:05,497 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/MasterData/WALs/jenkins-hbase3.apache.org,33223,1694032501568/jenkins-hbase3.apache.org%2C33223%2C1694032501568.1694032505475, exclude list is [], retry=0 2023-09-06T20:35:05,529 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46843,DS-1307a321-484a-4087-acdc-931715071dc3,DISK] 2023-09-06T20:35:05,537 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:346) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:120) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:430) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:424) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:120) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:482) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:477) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:489) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:397) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:35:05,612 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/MasterData/WALs/jenkins-hbase3.apache.org,33223,1694032501568/jenkins-hbase3.apache.org%2C33223%2C1694032501568.1694032505475 2023-09-06T20:35:05,612 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:46843,DS-1307a321-484a-4087-acdc-931715071dc3,DISK]] 2023-09-06T20:35:05,613 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:35:05,613 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:05,617 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:35:05,619 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:35:05,686 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:35:05,722 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-09-06T20:35:05,727 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:05,731 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:35:05,731 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:35:05,734 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-09-06T20:35:05,734 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:05,735 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:35:05,736 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:35:05,738 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-09-06T20:35:05,739 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:05,739 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:35:05,740 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:35:05,742 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-09-06T20:35:05,742 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:05,743 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:35:05,747 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:35:05,748 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:35:05,765 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-09-06T20:35:05,769 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:35:05,773 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:35:05,774 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=12040036960, jitterRate=0.12131582200527191}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-09-06T20:35:05,781 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-09-06T20:35:05,782 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-09-06T20:35:05,818 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@755c0a9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:05,874 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(860): No meta location available on zookeeper, skip migrating... 2023-09-06T20:35:05,894 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-09-06T20:35:05,894 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(563): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-09-06T20:35:05,897 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-09-06T20:35:05,899 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(583): Recovered RegionProcedureStore lease in 1 msec 2023-09-06T20:35:05,906 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(597): Loaded RegionProcedureStore in 6 msec 2023-09-06T20:35:05,906 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-09-06T20:35:05,942 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-09-06T20:35:05,959 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2023-09-06T20:35:05,961 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/balancer already deleted, retry=false 2023-09-06T20:35:05,964 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(150): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-09-06T20:35:05,965 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2023-09-06T20:35:05,966 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/normalizer already deleted, retry=false 2023-09-06T20:35:05,969 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-09-06T20:35:05,976 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2023-09-06T20:35:05,978 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/split already deleted, retry=false 2023-09-06T20:35:05,979 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2023-09-06T20:35:05,980 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/merge already deleted, retry=false 2023-09-06T20:35:05,996 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2023-09-06T20:35:05,997 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/snapshot-cleanup already deleted, retry=false 2023-09-06T20:35:06,002 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-09-06T20:35:06,002 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-09-06T20:35:06,003 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:35:06,003 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:35:06,003 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(804): Active/primary master=jenkins-hbase3.apache.org,33223,1694032501568, sessionid=0x100eefdf7f60000, setting cluster-up flag (Was=false) 2023-09-06T20:35:06,013 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-09-06T20:35:06,020 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:35:06,021 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:35:06,026 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2023-09-06T20:35:06,028 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,33223,1694032501568 2023-09-06T20:35:06,030 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:35:06,031 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:35:06,037 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2023-09-06T20:35:06,038 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,33223,1694032501568 2023-09-06T20:35:06,043 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:35:06,043 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:35:06,053 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2023-09-06T20:35:06,054 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,33223,1694032501568 2023-09-06T20:35:06,147 DEBUG [RS:0;jenkins-hbase3:36075 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:36075 2023-09-06T20:35:06,149 INFO [RS:0;jenkins-hbase3:36075 {}] regionserver.HRegionServer(997): ClusterId : dfcf656f-3e2c-409d-a4c9-cc1fca9f9955 2023-09-06T20:35:06,152 INFO [RS:0;jenkins-hbase3:36075 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-09-06T20:35:06,155 DEBUG [RS:0;jenkins-hbase3:36075 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-09-06T20:35:06,162 DEBUG [RS:0;jenkins-hbase3:36075 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-09-06T20:35:06,162 DEBUG [RS:0;jenkins-hbase3:36075 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-09-06T20:35:06,167 DEBUG [RS:0;jenkins-hbase3:36075 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-09-06T20:35:06,167 DEBUG [RS:0;jenkins-hbase3:36075 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-09-06T20:35:06,171 DEBUG [RS:0;jenkins-hbase3:36075 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-09-06T20:35:06,172 DEBUG [RS:0;jenkins-hbase3:36075 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4af57d15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:06,174 DEBUG [RS:0;jenkins-hbase3:36075 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21de2759, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-09-06T20:35:06,182 INFO [RS:0;jenkins-hbase3:36075 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-09-06T20:35:06,182 INFO [RS:0;jenkins-hbase3:36075 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-09-06T20:35:06,182 DEBUG [RS:0;jenkins-hbase3:36075 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-09-06T20:35:06,186 INFO [RS:0;jenkins-hbase3:36075 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,33223,1694032501568 with isa=jenkins-hbase3.apache.org/172.31.12.81:36075, startcode=1694032504561 2023-09-06T20:35:06,213 DEBUG [RS:0;jenkins-hbase3:36075 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-09-06T20:35:06,221 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-09-06T20:35:06,231 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-09-06T20:35:06,236 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-09-06T20:35:06,244 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,33223,1694032501568 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-09-06T20:35:06,250 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-06T20:35:06,250 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-06T20:35:06,251 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-06T20:35:06,251 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-06T20:35:06,251 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-09-06T20:35:06,251 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:06,252 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-09-06T20:35:06,252 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:06,253 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1694032536253 2023-09-06T20:35:06,257 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-09-06T20:35:06,262 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-09-06T20:35:06,264 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-09-06T20:35:06,264 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-09-06T20:35:06,272 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:06,273 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-09-06T20:35:06,275 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-09-06T20:35:06,276 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-09-06T20:35:06,277 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-09-06T20:35:06,278 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-09-06T20:35:06,278 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-09-06T20:35:06,279 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:06,282 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-09-06T20:35:06,287 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-09-06T20:35:06,288 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-09-06T20:35:06,290 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-09-06T20:35:06,291 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd 2023-09-06T20:35:06,298 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:59551, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-09-06T20:35:06,299 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7f1b5456 to 127.0.0.1:50423 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:35:06,312 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33223 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3181) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:590) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17393) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:35:06,318 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40543240, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:06,318 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-09-06T20:35:06,322 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-09-06T20:35:06,323 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-09-06T20:35:06,326 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1694032506325,5,FailOnTimeoutGroup] 2023-09-06T20:35:06,326 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1694032506326,5,FailOnTimeoutGroup] 2023-09-06T20:35:06,326 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:06,326 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1635): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-09-06T20:35:06,328 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:06,328 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:06,355 DEBUG [RS:0;jenkins-hbase3:36075 {}] regionserver.HRegionServer(3046): Master is not running yet 2023-09-06T20:35:06,355 WARN [RS:0;jenkins-hbase3:36075 {}] regionserver.HRegionServer(1088): reportForDuty failed; sleeping 100 ms and then retrying. 2023-09-06T20:35:06,456 INFO [RS:0;jenkins-hbase3:36075 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,33223,1694032501568 with isa=jenkins-hbase3.apache.org/172.31.12.81:36075, startcode=1694032504561 2023-09-06T20:35:06,461 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33223 {}] master.ServerManager(403): Registering regionserver=jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:06,472 DEBUG [RS:0;jenkins-hbase3:36075 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd 2023-09-06T20:35:06,472 DEBUG [RS:0;jenkins-hbase3:36075 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:36869 2023-09-06T20:35:06,472 DEBUG [RS:0;jenkins-hbase3:36075 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=38881 2023-09-06T20:35:06,478 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-09-06T20:35:06,479 DEBUG [RS:0;jenkins-hbase3:36075 {}] zookeeper.ZKUtil(111): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Set watcher on existing znode=/1/rs/jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:06,479 WARN [RS:0;jenkins-hbase3:36075 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-09-06T20:35:06,479 INFO [RS:0;jenkins-hbase3:36075 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-09-06T20:35:06,479 DEBUG [RS:0;jenkins-hbase3:36075 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/WALs/jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:06,482 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,36075,1694032504561] 2023-09-06T20:35:06,500 DEBUG [RS:0;jenkins-hbase3:36075 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-09-06T20:35:06,519 INFO [RS:0;jenkins-hbase3:36075 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-09-06T20:35:06,532 INFO [RS:0;jenkins-hbase3:36075 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-09-06T20:35:06,536 INFO [RS:0;jenkins-hbase3:36075 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-09-06T20:35:06,537 INFO [RS:0;jenkins-hbase3:36075 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:06,538 INFO [RS:0;jenkins-hbase3:36075 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-09-06T20:35:06,547 INFO [RS:0;jenkins-hbase3:36075 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:06,547 DEBUG [RS:0;jenkins-hbase3:36075 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:06,547 DEBUG [RS:0;jenkins-hbase3:36075 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:06,547 DEBUG [RS:0;jenkins-hbase3:36075 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:06,547 DEBUG [RS:0;jenkins-hbase3:36075 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:06,547 DEBUG [RS:0;jenkins-hbase3:36075 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:06,548 DEBUG [RS:0;jenkins-hbase3:36075 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-09-06T20:35:06,548 DEBUG [RS:0;jenkins-hbase3:36075 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:06,548 DEBUG [RS:0;jenkins-hbase3:36075 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:06,548 DEBUG [RS:0;jenkins-hbase3:36075 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:06,548 DEBUG [RS:0;jenkins-hbase3:36075 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:06,548 DEBUG [RS:0;jenkins-hbase3:36075 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:06,548 DEBUG [RS:0;jenkins-hbase3:36075 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-09-06T20:35:06,548 DEBUG [RS:0;jenkins-hbase3:36075 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-09-06T20:35:06,549 INFO [RS:0;jenkins-hbase3:36075 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:06,550 INFO [RS:0;jenkins-hbase3:36075 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:06,550 INFO [RS:0;jenkins-hbase3:36075 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:06,550 INFO [RS:0;jenkins-hbase3:36075 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:06,550 INFO [RS:0;jenkins-hbase3:36075 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,36075,1694032504561-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-09-06T20:35:06,565 INFO [RS:0;jenkins-hbase3:36075 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-09-06T20:35:06,568 INFO [RS:0;jenkins-hbase3:36075 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,36075,1694032504561-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:06,583 INFO [RS:0;jenkins-hbase3:36075 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,36075,1694032504561 started 2023-09-06T20:35:06,583 INFO [RS:0;jenkins-hbase3:36075 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,36075,1694032504561, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:36075, sessionid=0x100eefdf7f60001 2023-09-06T20:35:06,584 DEBUG [RS:0;jenkins-hbase3:36075 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-09-06T20:35:06,584 DEBUG [RS:0;jenkins-hbase3:36075 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:06,584 DEBUG [RS:0;jenkins-hbase3:36075 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,36075,1694032504561' 2023-09-06T20:35:06,584 DEBUG [RS:0;jenkins-hbase3:36075 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2023-09-06T20:35:06,585 DEBUG [RS:0;jenkins-hbase3:36075 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2023-09-06T20:35:06,585 DEBUG [RS:0;jenkins-hbase3:36075 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-09-06T20:35:06,586 DEBUG [RS:0;jenkins-hbase3:36075 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-09-06T20:35:06,586 DEBUG [RS:0;jenkins-hbase3:36075 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,36075,1694032504561' 2023-09-06T20:35:06,586 DEBUG [RS:0;jenkins-hbase3:36075 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:35:06,586 DEBUG [RS:0;jenkins-hbase3:36075 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:35:06,587 INFO [RS:0;jenkins-hbase3:36075 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-09-06T20:35:06,587 DEBUG [RS:0;jenkins-hbase3:36075 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-09-06T20:35:06,587 DEBUG [RS:0;jenkins-hbase3:36075 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-09-06T20:35:06,587 DEBUG [RS:0;jenkins-hbase3:36075 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:06,587 DEBUG [RS:0;jenkins-hbase3:36075 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,36075,1694032504561' 2023-09-06T20:35:06,587 DEBUG [RS:0;jenkins-hbase3:36075 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2023-09-06T20:35:06,588 DEBUG [RS:0;jenkins-hbase3:36075 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2023-09-06T20:35:06,588 DEBUG [RS:0;jenkins-hbase3:36075 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-09-06T20:35:06,588 INFO [RS:0;jenkins-hbase3:36075 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-09-06T20:35:06,588 INFO [RS:0;jenkins-hbase3:36075 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-09-06T20:35:06,703 INFO [RS:0;jenkins-hbase3:36075 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-09-06T20:35:06,707 INFO [RS:0;jenkins-hbase3:36075 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C36075%2C1694032504561, suffix=, logDir=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/WALs/jenkins-hbase3.apache.org,36075,1694032504561, archiveDir=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/oldWALs, maxLogs=32 2023-09-06T20:35:06,711 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:06,714 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-09-06T20:35:06,716 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-09-06T20:35:06,717 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:06,718 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:35:06,718 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-09-06T20:35:06,720 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-09-06T20:35:06,720 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:06,721 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:35:06,722 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-09-06T20:35:06,724 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-09-06T20:35:06,724 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:06,725 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:35:06,727 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/hbase/meta/1588230740 2023-09-06T20:35:06,727 DEBUG [RS:0;jenkins-hbase3:36075 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/WALs/jenkins-hbase3.apache.org,36075,1694032504561/jenkins-hbase3.apache.org%2C36075%2C1694032504561.1694032506709, exclude list is [], retry=0 2023-09-06T20:35:06,728 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/hbase/meta/1588230740 2023-09-06T20:35:06,734 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46843,DS-1307a321-484a-4087-acdc-931715071dc3,DISK] 2023-09-06T20:35:06,736 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-09-06T20:35:06,739 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-09-06T20:35:06,745 INFO [RS:0;jenkins-hbase3:36075 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/WALs/jenkins-hbase3.apache.org,36075,1694032504561/jenkins-hbase3.apache.org%2C36075%2C1694032504561.1694032506709 2023-09-06T20:35:06,746 DEBUG [RS:0;jenkins-hbase3:36075 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:46843,DS-1307a321-484a-4087-acdc-931715071dc3,DISK]] 2023-09-06T20:35:06,746 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:35:06,747 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11405955520, jitterRate=0.062262386083602905}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-09-06T20:35:06,749 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-09-06T20:35:06,749 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-09-06T20:35:06,749 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-09-06T20:35:06,750 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-09-06T20:35:06,750 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-09-06T20:35:06,750 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-09-06T20:35:06,755 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-09-06T20:35:06,755 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-09-06T20:35:06,760 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-09-06T20:35:06,760 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-09-06T20:35:06,774 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-09-06T20:35:06,793 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-09-06T20:35:06,799 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-09-06T20:35:06,951 DEBUG [jenkins-hbase3:33223 {}] assignment.AssignmentManager(2303): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-09-06T20:35:06,960 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:06,969 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,36075,1694032504561, state=OPENING 2023-09-06T20:35:06,978 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-09-06T20:35:06,982 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:35:06,982 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:35:06,983 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-09-06T20:35:06,983 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-09-06T20:35:06,987 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,36075,1694032504561}] 2023-09-06T20:35:07,179 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(721): New admin connection to jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:07,182 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2023-09-06T20:35:07,188 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:52328, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-09-06T20:35:07,202 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-09-06T20:35:07,203 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-09-06T20:35:07,203 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-09-06T20:35:07,207 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C36075%2C1694032504561.meta, suffix=.meta, logDir=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/WALs/jenkins-hbase3.apache.org,36075,1694032504561, archiveDir=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/oldWALs, maxLogs=32 2023-09-06T20:35:07,219 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/WALs/jenkins-hbase3.apache.org,36075,1694032504561/jenkins-hbase3.apache.org%2C36075%2C1694032504561.meta.1694032507208.meta, exclude list is [], retry=0 2023-09-06T20:35:07,224 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46843,DS-1307a321-484a-4087-acdc-931715071dc3,DISK] 2023-09-06T20:35:07,227 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/WALs/jenkins-hbase3.apache.org,36075,1694032504561/jenkins-hbase3.apache.org%2C36075%2C1694032504561.meta.1694032507208.meta 2023-09-06T20:35:07,228 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:46843,DS-1307a321-484a-4087-acdc-931715071dc3,DISK]] 2023-09-06T20:35:07,228 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:35:07,233 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:35:07,234 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-09-06T20:35:07,250 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-09-06T20:35:07,250 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-09-06T20:35:07,255 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-09-06T20:35:07,255 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:07,255 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-09-06T20:35:07,255 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-09-06T20:35:07,258 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-09-06T20:35:07,260 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-09-06T20:35:07,260 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:07,261 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:35:07,261 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-09-06T20:35:07,262 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-09-06T20:35:07,262 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:07,263 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:35:07,263 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-09-06T20:35:07,264 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-09-06T20:35:07,264 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:07,265 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:35:07,267 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/hbase/meta/1588230740 2023-09-06T20:35:07,269 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/hbase/meta/1588230740 2023-09-06T20:35:07,272 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-09-06T20:35:07,274 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-09-06T20:35:07,276 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11714240800, jitterRate=0.09097369015216827}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-09-06T20:35:07,277 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-09-06T20:35:07,289 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1694032507170 2023-09-06T20:35:07,309 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:meta,,1.1588230740 2023-09-06T20:35:07,310 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-09-06T20:35:07,311 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:07,313 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,36075,1694032504561, state=OPEN 2023-09-06T20:35:07,316 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-09-06T20:35:07,316 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-09-06T20:35:07,316 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-09-06T20:35:07,316 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-09-06T20:35:07,320 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=3, resume processing ppid=2 2023-09-06T20:35:07,321 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,36075,1694032504561 in 329 msec 2023-09-06T20:35:07,471 INFO [AsyncFSWAL-0-hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/MasterData {}] wal.AbstractFSWAL(1174): Slow sync cost: 140 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46843,DS-1307a321-484a-4087-acdc-931715071dc3,DISK]] 2023-09-06T20:35:07,472 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=2, resume processing ppid=1 2023-09-06T20:35:07,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 548 msec 2023-09-06T20:35:07,480 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.3540 sec 2023-09-06T20:35:07,480 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1063): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1694032507480, completionTime=-1 2023-09-06T20:35:07,480 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(830): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-09-06T20:35:07,480 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1629): Joining cluster... 2023-09-06T20:35:07,551 DEBUG [hconnection-0x1ea85617-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:07,556 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:52330, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:07,573 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1641): Number of RegionServers=1 2023-09-06T20:35:07,573 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1694032567573 2023-09-06T20:35:07,573 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1694032627573 2023-09-06T20:35:07,573 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1648): Joined the cluster in 92 msec 2023-09-06T20:35:07,600 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,33223,1694032501568-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:07,600 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,33223,1694032501568-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:07,600 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,33223,1694032501568-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:07,602 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:33223, period=300000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:07,603 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:07,610 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-09-06T20:35:07,624 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-09-06T20:35:07,625 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2368): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-09-06T20:35:07,634 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-09-06T20:35:07,638 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:35:07,641 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:07,643 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:35:08,065 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => cd7d22af9ee99f89485a5c716f51a037, NAME => 'hbase:namespace,,1694032507625.cd7d22af9ee99f89485a5c716f51a037.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd 2023-09-06T20:35:08,479 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1694032507625.cd7d22af9ee99f89485a5c716f51a037.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:08,479 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing cd7d22af9ee99f89485a5c716f51a037, disabling compactions & flushes 2023-09-06T20:35:08,479 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1694032507625.cd7d22af9ee99f89485a5c716f51a037. 2023-09-06T20:35:08,479 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1694032507625.cd7d22af9ee99f89485a5c716f51a037. 2023-09-06T20:35:08,479 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1694032507625.cd7d22af9ee99f89485a5c716f51a037. after waiting 0 ms 2023-09-06T20:35:08,479 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1694032507625.cd7d22af9ee99f89485a5c716f51a037. 2023-09-06T20:35:08,479 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1694032507625.cd7d22af9ee99f89485a5c716f51a037. 2023-09-06T20:35:08,479 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for cd7d22af9ee99f89485a5c716f51a037: 2023-09-06T20:35:08,482 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:35:08,496 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1694032507625.cd7d22af9ee99f89485a5c716f51a037.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1694032508484"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032508484"}]},"ts":"1694032508484"} 2023-09-06T20:35:08,534 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:35:08,537 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:35:08,542 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032508537"}]},"ts":"1694032508537"} 2023-09-06T20:35:08,546 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-09-06T20:35:08,553 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=cd7d22af9ee99f89485a5c716f51a037, ASSIGN}] 2023-09-06T20:35:08,555 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=cd7d22af9ee99f89485a5c716f51a037, ASSIGN 2023-09-06T20:35:08,557 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=cd7d22af9ee99f89485a5c716f51a037, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36075,1694032504561; forceNewPlan=false, retain=false 2023-09-06T20:35:08,708 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=cd7d22af9ee99f89485a5c716f51a037, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:08,712 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure cd7d22af9ee99f89485a5c716f51a037, server=jenkins-hbase3.apache.org,36075,1694032504561}] 2023-09-06T20:35:08,872 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1694032507625.cd7d22af9ee99f89485a5c716f51a037. 2023-09-06T20:35:08,873 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => cd7d22af9ee99f89485a5c716f51a037, NAME => 'hbase:namespace,,1694032507625.cd7d22af9ee99f89485a5c716f51a037.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:35:08,874 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:35:08,874 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace cd7d22af9ee99f89485a5c716f51a037 2023-09-06T20:35:08,875 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1694032507625.cd7d22af9ee99f89485a5c716f51a037.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:08,875 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for cd7d22af9ee99f89485a5c716f51a037 2023-09-06T20:35:08,875 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for cd7d22af9ee99f89485a5c716f51a037 2023-09-06T20:35:08,877 INFO [StoreOpener-cd7d22af9ee99f89485a5c716f51a037-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region cd7d22af9ee99f89485a5c716f51a037 2023-09-06T20:35:08,879 INFO [StoreOpener-cd7d22af9ee99f89485a5c716f51a037-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cd7d22af9ee99f89485a5c716f51a037 columnFamilyName info 2023-09-06T20:35:08,879 DEBUG [StoreOpener-cd7d22af9ee99f89485a5c716f51a037-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:08,880 INFO [StoreOpener-cd7d22af9ee99f89485a5c716f51a037-1 {}] regionserver.HStore(324): Store=cd7d22af9ee99f89485a5c716f51a037/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:35:08,882 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/hbase/namespace/cd7d22af9ee99f89485a5c716f51a037 2023-09-06T20:35:08,883 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/hbase/namespace/cd7d22af9ee99f89485a5c716f51a037 2023-09-06T20:35:08,886 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for cd7d22af9ee99f89485a5c716f51a037 2023-09-06T20:35:08,890 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/hbase/namespace/cd7d22af9ee99f89485a5c716f51a037/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:35:08,891 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened cd7d22af9ee99f89485a5c716f51a037; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11329891520, jitterRate=0.05517837405204773}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:35:08,892 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for cd7d22af9ee99f89485a5c716f51a037: 2023-09-06T20:35:08,894 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:namespace,,1694032507625.cd7d22af9ee99f89485a5c716f51a037., pid=6, masterSystemTime=1694032508866 2023-09-06T20:35:08,898 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:namespace,,1694032507625.cd7d22af9ee99f89485a5c716f51a037. 2023-09-06T20:35:08,898 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1694032507625.cd7d22af9ee99f89485a5c716f51a037. 2023-09-06T20:35:08,899 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=cd7d22af9ee99f89485a5c716f51a037, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:08,909 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=6, resume processing ppid=5 2023-09-06T20:35:08,909 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure cd7d22af9ee99f89485a5c716f51a037, server=jenkins-hbase3.apache.org,36075,1694032504561 in 193 msec 2023-09-06T20:35:08,913 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=5, resume processing ppid=4 2023-09-06T20:35:08,913 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=cd7d22af9ee99f89485a5c716f51a037, ASSIGN in 356 msec 2023-09-06T20:35:08,914 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:35:08,915 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032508914"}]},"ts":"1694032508914"} 2023-09-06T20:35:08,918 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-09-06T20:35:08,921 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:35:08,924 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 1.2950 sec 2023-09-06T20:35:08,939 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Set watcher on znode that does not yet exist, /1/namespace 2023-09-06T20:35:08,940 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/namespace 2023-09-06T20:35:08,940 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:35:08,941 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:35:08,997 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-09-06T20:35:09,016 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:35:09,022 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 37 msec 2023-09-06T20:35:09,035 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-09-06T20:35:09,046 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:35:09,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 15 msec 2023-09-06T20:35:09,062 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/default 2023-09-06T20:35:09,064 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/hbase 2023-09-06T20:35:09,066 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1193): Master has completed initialization 4.379sec 2023-09-06T20:35:09,068 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-09-06T20:35:09,070 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-09-06T20:35:09,071 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-09-06T20:35:09,072 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-09-06T20:35:09,072 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-09-06T20:35:09,075 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,33223,1694032501568-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-09-06T20:35:09,075 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,33223,1694032501568-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-09-06T20:35:09,080 DEBUG [Listener at localhost/43069 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3813c1aa to 127.0.0.1:50423 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:35:09,085 DEBUG [Listener at localhost/43069 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@344d2abb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:09,094 DEBUG [Listener at localhost/43069 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-09-06T20:35:09,094 DEBUG [Listener at localhost/43069 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-09-06T20:35:09,096 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1296): Balancer post startup initialization complete, took 0 seconds 2023-09-06T20:35:09,098 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-09-06T20:35:09,112 DEBUG [hconnection-0x7de0eab9-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:09,123 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:52338, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:09,134 INFO [Listener at localhost/43069 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,33223,1694032501568 2023-09-06T20:35:09,147 INFO [Listener at localhost/43069 {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/hadoop.log.dir so I do NOT create it in target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792 2023-09-06T20:35:09,147 INFO [Listener at localhost/43069 {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/hadoop.tmp.dir so I do NOT create it in target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792 2023-09-06T20:35:09,147 DEBUG [Listener at localhost/43069 {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792 2023-09-06T20:35:09,147 INFO [Listener at localhost/43069 {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/cluster_c0c6cff4-aaf0-dadf-2d25-742d9edde910, deleteOnExit=true 2023-09-06T20:35:09,147 INFO [Listener at localhost/43069 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/test.cache.data in system properties and HBase conf 2023-09-06T20:35:09,147 INFO [Listener at localhost/43069 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/hadoop.tmp.dir in system properties and HBase conf 2023-09-06T20:35:09,147 INFO [Listener at localhost/43069 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/hadoop.log.dir in system properties and HBase conf 2023-09-06T20:35:09,147 INFO [Listener at localhost/43069 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/mapreduce.cluster.local.dir in system properties and HBase conf 2023-09-06T20:35:09,148 INFO [Listener at localhost/43069 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-09-06T20:35:09,148 INFO [Listener at localhost/43069 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-09-06T20:35:09,148 DEBUG [Listener at localhost/43069 {}] fs.HFileSystem(308): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-09-06T20:35:09,148 INFO [Listener at localhost/43069 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-09-06T20:35:09,148 INFO [Listener at localhost/43069 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-09-06T20:35:09,148 INFO [Listener at localhost/43069 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-09-06T20:35:09,148 INFO [Listener at localhost/43069 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-09-06T20:35:09,149 INFO [Listener at localhost/43069 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-09-06T20:35:09,149 INFO [Listener at localhost/43069 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-09-06T20:35:09,149 INFO [Listener at localhost/43069 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-09-06T20:35:09,149 INFO [Listener at localhost/43069 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/dfs.journalnode.edits.dir in system properties and HBase conf 2023-09-06T20:35:09,149 INFO [Listener at localhost/43069 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-09-06T20:35:09,149 INFO [Listener at localhost/43069 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/nfs.dump.dir in system properties and HBase conf 2023-09-06T20:35:09,149 INFO [Listener at localhost/43069 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/java.io.tmpdir in system properties and HBase conf 2023-09-06T20:35:09,149 INFO [Listener at localhost/43069 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/dfs.journalnode.edits.dir in system properties and HBase conf 2023-09-06T20:35:09,149 INFO [Listener at localhost/43069 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-09-06T20:35:09,149 INFO [Listener at localhost/43069 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-09-06T20:35:09,163 WARN [Listener at localhost/43069 {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-09-06T20:35:09,163 WARN [Listener at localhost/43069 {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-09-06T20:35:09,215 WARN [Listener at localhost/43069 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:35:09,219 INFO [Listener at localhost/43069 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:35:09,226 INFO [Listener at localhost/43069 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/java.io.tmpdir/Jetty_localhost_32965_hdfs____.rldavx/webapp 2023-09-06T20:35:09,343 INFO [Listener at localhost/43069 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:32965 2023-09-06T20:35:09,356 WARN [Listener at localhost/43069 {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-09-06T20:35:09,356 WARN [Listener at localhost/43069 {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-09-06T20:35:09,402 WARN [Listener at localhost/41733 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-09-06T20:35:09,437 WARN [Listener at localhost/41733 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-09-06T20:35:09,440 WARN [Listener at localhost/41733 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:35:09,442 INFO [Listener at localhost/41733 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:35:09,449 INFO [Listener at localhost/41733 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/java.io.tmpdir/Jetty_localhost_42259_datanode____vo6xa7/webapp 2023-09-06T20:35:09,575 INFO [Listener at localhost/41733 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:42259 2023-09-06T20:35:09,584 WARN [Listener at localhost/34535 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-09-06T20:35:09,634 WARN [Listener at localhost/34535 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-09-06T20:35:09,636 WARN [Listener at localhost/34535 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:35:09,639 INFO [Listener at localhost/34535 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:35:09,647 INFO [Listener at localhost/34535 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/java.io.tmpdir/Jetty_localhost_43933_datanode____uhwbxd/webapp 2023-09-06T20:35:09,683 WARN [Thread-202 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-09-06T20:35:09,722 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xa02ad68579ff9865: Processing first storage report for DS-71098be5-7b27-4f7d-ab7a-e0614e889cd8 from datanode bf782b2f-52a2-4551-9a69-02d41579503c 2023-09-06T20:35:09,722 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xa02ad68579ff9865: from storage DS-71098be5-7b27-4f7d-ab7a-e0614e889cd8 node DatanodeRegistration(127.0.0.1:38883, datanodeUuid=bf782b2f-52a2-4551-9a69-02d41579503c, infoPort=45567, infoSecurePort=0, ipcPort=34535, storageInfo=lv=-57;cid=testClusterID;nsid=1304650494;c=1694032509166), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2023-09-06T20:35:09,722 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xa02ad68579ff9865: Processing first storage report for DS-bfe0e0fe-6b54-423a-baef-08541a146d23 from datanode bf782b2f-52a2-4551-9a69-02d41579503c 2023-09-06T20:35:09,722 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xa02ad68579ff9865: from storage DS-bfe0e0fe-6b54-423a-baef-08541a146d23 node DatanodeRegistration(127.0.0.1:38883, datanodeUuid=bf782b2f-52a2-4551-9a69-02d41579503c, infoPort=45567, infoSecurePort=0, ipcPort=34535, storageInfo=lv=-57;cid=testClusterID;nsid=1304650494;c=1694032509166), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-09-06T20:35:09,761 INFO [Listener at localhost/34535 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:43933 2023-09-06T20:35:09,769 WARN [Listener at localhost/40951 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-09-06T20:35:09,796 WARN [Listener at localhost/40951 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-09-06T20:35:09,799 WARN [Listener at localhost/40951 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:35:09,800 INFO [Listener at localhost/40951 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:35:09,806 INFO [Listener at localhost/40951 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/java.io.tmpdir/Jetty_localhost_38513_datanode____oyd7cp/webapp 2023-09-06T20:35:09,846 WARN [Thread-237 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-09-06T20:35:09,876 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xcd542ed42e8429a4: Processing first storage report for DS-e43d6617-42bd-47f8-b767-588483aeb5af from datanode f5c319ce-5a8a-47aa-bca3-cdfda495d7b7 2023-09-06T20:35:09,876 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xcd542ed42e8429a4: from storage DS-e43d6617-42bd-47f8-b767-588483aeb5af node DatanodeRegistration(127.0.0.1:44329, datanodeUuid=f5c319ce-5a8a-47aa-bca3-cdfda495d7b7, infoPort=34281, infoSecurePort=0, ipcPort=40951, storageInfo=lv=-57;cid=testClusterID;nsid=1304650494;c=1694032509166), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2023-09-06T20:35:09,876 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xcd542ed42e8429a4: Processing first storage report for DS-7543fc23-d5a3-4313-8047-5e7b81c19b3f from datanode f5c319ce-5a8a-47aa-bca3-cdfda495d7b7 2023-09-06T20:35:09,876 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xcd542ed42e8429a4: from storage DS-7543fc23-d5a3-4313-8047-5e7b81c19b3f node DatanodeRegistration(127.0.0.1:44329, datanodeUuid=f5c319ce-5a8a-47aa-bca3-cdfda495d7b7, infoPort=34281, infoSecurePort=0, ipcPort=40951, storageInfo=lv=-57;cid=testClusterID;nsid=1304650494;c=1694032509166), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-09-06T20:35:09,915 INFO [Listener at localhost/40951 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:38513 2023-09-06T20:35:09,923 WARN [Listener at localhost/39521 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-09-06T20:35:10,009 WARN [Thread-271 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-09-06T20:35:10,035 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x8e3e057b3cac2610: Processing first storage report for DS-5c3dda06-912e-4de1-b561-0f5c1ff9ef54 from datanode f2077e27-413a-4359-9405-99f59d43a4d4 2023-09-06T20:35:10,035 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x8e3e057b3cac2610: from storage DS-5c3dda06-912e-4de1-b561-0f5c1ff9ef54 node DatanodeRegistration(127.0.0.1:46471, datanodeUuid=f2077e27-413a-4359-9405-99f59d43a4d4, infoPort=41461, infoSecurePort=0, ipcPort=39521, storageInfo=lv=-57;cid=testClusterID;nsid=1304650494;c=1694032509166), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2023-09-06T20:35:10,035 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x8e3e057b3cac2610: Processing first storage report for DS-85f862b1-bfbc-4d28-8674-60ec20a55764 from datanode f2077e27-413a-4359-9405-99f59d43a4d4 2023-09-06T20:35:10,035 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x8e3e057b3cac2610: from storage DS-85f862b1-bfbc-4d28-8674-60ec20a55764 node DatanodeRegistration(127.0.0.1:46471, datanodeUuid=f2077e27-413a-4359-9405-99f59d43a4d4, infoPort=41461, infoSecurePort=0, ipcPort=39521, storageInfo=lv=-57;cid=testClusterID;nsid=1304650494;c=1694032509166), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-09-06T20:35:10,138 DEBUG [Listener at localhost/39521 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792 2023-09-06T20:35:10,138 INFO [Listener at localhost/39521 {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-09-06T20:35:10,139 INFO [Listener at localhost/39521 {}] hbase.HBaseTestingUtility(1143): NOT STARTING DFS 2023-09-06T20:35:10,139 INFO [Listener at localhost/39521 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:35:10,141 INFO [Listener at localhost/39521 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:35:10,564 INFO [Listener at localhost/39521 {}] util.FSUtils(461): Created version file at hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e with version=8 2023-09-06T20:35:10,564 INFO [Listener at localhost/39521 {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/hbase-staging 2023-09-06T20:35:10,565 DEBUG [Listener at localhost/39521 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-09-06T20:35:10,565 DEBUG [Listener at localhost/39521 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-09-06T20:35:10,565 DEBUG [Listener at localhost/39521 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-09-06T20:35:10,565 DEBUG [Listener at localhost/39521 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-09-06T20:35:10,566 INFO [Listener at localhost/39521 {}] client.ConnectionUtils(128): master/jenkins-hbase3:0 server-side Connection retries=45 2023-09-06T20:35:10,566 INFO [Listener at localhost/39521 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-06T20:35:10,566 INFO [Listener at localhost/39521 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-09-06T20:35:10,566 INFO [Listener at localhost/39521 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-09-06T20:35:10,567 INFO [Listener at localhost/39521 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-06T20:35:10,567 INFO [Listener at localhost/39521 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-09-06T20:35:10,567 INFO [Listener at localhost/39521 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-09-06T20:35:10,567 INFO [Listener at localhost/39521 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-09-06T20:35:10,568 INFO [Listener at localhost/39521 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:41475 2023-09-06T20:35:10,569 INFO [Listener at localhost/39521 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:35:10,570 INFO [Listener at localhost/39521 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:35:10,571 INFO [Listener at localhost/39521 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:41475 connecting to ZooKeeper ensemble=127.0.0.1:50423 2023-09-06T20:35:10,574 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): master:414750x0, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-09-06T20:35:10,575 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:41475-0x100eefdf7f60004 connected 2023-09-06T20:35:10,587 DEBUG [Listener at localhost/39521 {}] zookeeper.ZKUtil(113): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Set watcher on znode that does not yet exist, /2/master 2023-09-06T20:35:10,588 DEBUG [Listener at localhost/39521 {}] zookeeper.ZKUtil(113): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Set watcher on znode that does not yet exist, /2/running 2023-09-06T20:35:10,588 DEBUG [Listener at localhost/39521 {}] zookeeper.ZKUtil(113): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Set watcher on znode that does not yet exist, /2/acl 2023-09-06T20:35:10,589 DEBUG [Listener at localhost/39521 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41475 2023-09-06T20:35:10,589 DEBUG [Listener at localhost/39521 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41475 2023-09-06T20:35:10,589 DEBUG [Listener at localhost/39521 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41475 2023-09-06T20:35:10,591 DEBUG [Listener at localhost/39521 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41475 2023-09-06T20:35:10,591 DEBUG [Listener at localhost/39521 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41475 2023-09-06T20:35:10,597 INFO [Listener at localhost/39521 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-09-06T20:35:10,598 INFO [Listener at localhost/39521 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-09-06T20:35:10,598 INFO [Listener at localhost/39521 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-09-06T20:35:10,598 INFO [Listener at localhost/39521 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-09-06T20:35:10,598 INFO [Listener at localhost/39521 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-09-06T20:35:10,598 INFO [Listener at localhost/39521 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-09-06T20:35:10,598 INFO [Listener at localhost/39521 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-09-06T20:35:10,600 INFO [Listener at localhost/39521 {}] http.HttpServer(1219): Jetty bound to port 45621 2023-09-06T20:35:10,600 INFO [Listener at localhost/39521 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-09-06T20:35:10,602 INFO [Listener at localhost/39521 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:35:10,602 INFO [Listener at localhost/39521 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@20cfc1d8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/hadoop.log.dir/,AVAILABLE} 2023-09-06T20:35:10,602 INFO [Listener at localhost/39521 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:35:10,602 INFO [Listener at localhost/39521 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@1f633e42{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-09-06T20:35:10,611 INFO [Listener at localhost/39521 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-09-06T20:35:10,612 INFO [Listener at localhost/39521 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-09-06T20:35:10,612 INFO [Listener at localhost/39521 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-09-06T20:35:10,613 INFO [Listener at localhost/39521 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2023-09-06T20:35:10,613 INFO [Listener at localhost/39521 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:35:10,706 INFO [Listener at localhost/39521 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@2ead5011{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-09-06T20:35:10,707 INFO [Listener at localhost/39521 {}] server.AbstractConnector(333): Started ServerConnector@31f32a89{HTTP/1.1, (http/1.1)}{0.0.0.0:45621} 2023-09-06T20:35:10,707 INFO [Listener at localhost/39521 {}] server.Server(415): Started @14546ms 2023-09-06T20:35:10,707 INFO [Listener at localhost/39521 {}] master.HMaster(485): hbase.rootdir=hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e, hbase.cluster.distributed=false 2023-09-06T20:35:10,721 INFO [Listener at localhost/39521 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase3:0 server-side Connection retries=45 2023-09-06T20:35:10,721 INFO [Listener at localhost/39521 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-06T20:35:10,721 INFO [Listener at localhost/39521 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-09-06T20:35:10,722 INFO [Listener at localhost/39521 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-09-06T20:35:10,722 INFO [Listener at localhost/39521 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-06T20:35:10,722 INFO [Listener at localhost/39521 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-09-06T20:35:10,722 INFO [Listener at localhost/39521 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-09-06T20:35:10,722 INFO [Listener at localhost/39521 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-09-06T20:35:10,724 INFO [Listener at localhost/39521 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:39115 2023-09-06T20:35:10,724 INFO [Listener at localhost/39521 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-09-06T20:35:10,726 DEBUG [Listener at localhost/39521 {}] mob.MobFileCache(120): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-09-06T20:35:10,727 INFO [Listener at localhost/39521 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:35:10,728 INFO [Listener at localhost/39521 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:35:10,730 INFO [Listener at localhost/39521 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:39115 connecting to ZooKeeper ensemble=127.0.0.1:50423 2023-09-06T20:35:10,733 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): regionserver:391150x0, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-09-06T20:35:10,735 DEBUG [Listener at localhost/39521 {}] zookeeper.ZKUtil(113): regionserver:391150x0, quorum=127.0.0.1:50423, baseZNode=/2 Set watcher on znode that does not yet exist, /2/master 2023-09-06T20:35:10,735 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:39115-0x100eefdf7f60005 connected 2023-09-06T20:35:10,735 DEBUG [Listener at localhost/39521 {}] zookeeper.ZKUtil(113): regionserver:39115-0x100eefdf7f60005, quorum=127.0.0.1:50423, baseZNode=/2 Set watcher on znode that does not yet exist, /2/running 2023-09-06T20:35:10,736 DEBUG [Listener at localhost/39521 {}] zookeeper.ZKUtil(113): regionserver:39115-0x100eefdf7f60005, quorum=127.0.0.1:50423, baseZNode=/2 Set watcher on znode that does not yet exist, /2/acl 2023-09-06T20:35:10,736 DEBUG [Listener at localhost/39521 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39115 2023-09-06T20:35:10,737 DEBUG [Listener at localhost/39521 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39115 2023-09-06T20:35:10,738 DEBUG [Listener at localhost/39521 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39115 2023-09-06T20:35:10,739 DEBUG [Listener at localhost/39521 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39115 2023-09-06T20:35:10,742 DEBUG [Listener at localhost/39521 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39115 2023-09-06T20:35:10,746 INFO [Listener at localhost/39521 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-09-06T20:35:10,746 INFO [Listener at localhost/39521 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-09-06T20:35:10,746 INFO [Listener at localhost/39521 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-09-06T20:35:10,747 INFO [Listener at localhost/39521 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-09-06T20:35:10,747 INFO [Listener at localhost/39521 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-09-06T20:35:10,747 INFO [Listener at localhost/39521 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-09-06T20:35:10,747 INFO [Listener at localhost/39521 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-09-06T20:35:10,748 INFO [Listener at localhost/39521 {}] http.HttpServer(1219): Jetty bound to port 45143 2023-09-06T20:35:10,748 INFO [Listener at localhost/39521 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-09-06T20:35:10,754 INFO [Listener at localhost/39521 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:35:10,754 INFO [Listener at localhost/39521 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@6bbaa492{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/hadoop.log.dir/,AVAILABLE} 2023-09-06T20:35:10,754 INFO [Listener at localhost/39521 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:35:10,754 INFO [Listener at localhost/39521 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@5286da90{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-09-06T20:35:10,762 INFO [Listener at localhost/39521 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-09-06T20:35:10,763 INFO [Listener at localhost/39521 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-09-06T20:35:10,763 INFO [Listener at localhost/39521 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-09-06T20:35:10,763 INFO [Listener at localhost/39521 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-09-06T20:35:10,764 INFO [Listener at localhost/39521 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:35:10,765 INFO [Listener at localhost/39521 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@1c95151d{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-09-06T20:35:10,767 INFO [Listener at localhost/39521 {}] server.AbstractConnector(333): Started ServerConnector@2e1a589d{HTTP/1.1, (http/1.1)}{0.0.0.0:45143} 2023-09-06T20:35:10,767 INFO [Listener at localhost/39521 {}] server.Server(415): Started @14605ms 2023-09-06T20:35:10,770 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-09-06T20:35:10,777 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@fb7fdda{HTTP/1.1, (http/1.1)}{0.0.0.0:42639} 2023-09-06T20:35:10,777 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(415): Started @14615ms 2023-09-06T20:35:10,777 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2388): Adding backup master ZNode /2/backup-masters/jenkins-hbase3.apache.org,41475,1694032510566 2023-09-06T20:35:10,780 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39115-0x100eefdf7f60005, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/backup-masters 2023-09-06T20:35:10,780 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/backup-masters 2023-09-06T20:35:10,782 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Set watcher on existing znode=/2/backup-masters/jenkins-hbase3.apache.org,41475,1694032510566 2023-09-06T20:35:10,784 DEBUG [M:0;jenkins-hbase3:41475 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:41475 2023-09-06T20:35:10,785 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/2/master 2023-09-06T20:35:10,785 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39115-0x100eefdf7f60005, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/2/master 2023-09-06T20:35:10,785 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-09-06T20:35:10,785 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39115-0x100eefdf7f60005, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-09-06T20:35:10,787 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Set watcher on existing znode=/2/master 2023-09-06T20:35:10,788 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Set watcher on existing znode=/2/master 2023-09-06T20:35:10,788 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /2/backup-masters/jenkins-hbase3.apache.org,41475,1694032510566 from backup master directory 2023-09-06T20:35:10,790 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/2/backup-masters/jenkins-hbase3.apache.org,41475,1694032510566 2023-09-06T20:35:10,790 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39115-0x100eefdf7f60005, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/backup-masters 2023-09-06T20:35:10,790 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/backup-masters 2023-09-06T20:35:10,790 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-09-06T20:35:10,790 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,41475,1694032510566 2023-09-06T20:35:10,819 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(610): Created cluster ID file at hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/hbase.id with ID: 839ab22e-31ed-4e78-b1ec-8e29bf999637 2023-09-06T20:35:10,836 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:35:10,839 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-09-06T20:35:10,839 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39115-0x100eefdf7f60005, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-09-06T20:35:11,252 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:35:11,253 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-09-06T20:35:11,254 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-09-06T20:35:11,673 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/MasterData/data/master/store 2023-09-06T20:35:12,089 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:12,089 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-09-06T20:35:12,089 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:35:12,089 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:35:12,089 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-09-06T20:35:12,089 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:35:12,089 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:35:12,089 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-09-06T20:35:12,091 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/MasterData/data/master/store/.initializing 2023-09-06T20:35:12,091 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:41733/tmp/wal/MasterData/WALs/jenkins-hbase3.apache.org,41475,1694032510566 2023-09-06T20:35:12,092 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-09-06T20:35:12,095 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C41475%2C1694032510566, suffix=, logDir=hdfs://localhost:41733/tmp/wal/MasterData/WALs/jenkins-hbase3.apache.org,41475,1694032510566, archiveDir=hdfs://localhost:41733/tmp/wal/MasterData/oldWALs, maxLogs=10 2023-09-06T20:35:12,106 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /tmp/wal/MasterData/WALs/jenkins-hbase3.apache.org,41475,1694032510566/jenkins-hbase3.apache.org%2C41475%2C1694032510566.1694032512095, exclude list is [], retry=0 2023-09-06T20:35:12,110 DEBUG [RS-EventLoopGroup-6-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38883,DS-71098be5-7b27-4f7d-ab7a-e0614e889cd8,DISK] 2023-09-06T20:35:12,114 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /tmp/wal/MasterData/WALs/jenkins-hbase3.apache.org,41475,1694032510566/jenkins-hbase3.apache.org%2C41475%2C1694032510566.1694032512095 2023-09-06T20:35:12,114 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38883,DS-71098be5-7b27-4f7d-ab7a-e0614e889cd8,DISK]] 2023-09-06T20:35:12,114 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:35:12,114 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:12,114 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:35:12,115 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:35:12,117 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:35:12,119 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-09-06T20:35:12,119 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:12,120 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:35:12,120 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:35:12,122 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-09-06T20:35:12,123 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:12,123 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:35:12,124 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:35:12,126 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-09-06T20:35:12,126 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:12,126 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:35:12,127 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:35:12,128 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-09-06T20:35:12,128 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:12,129 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:35:12,130 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:35:12,131 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:35:12,131 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41733/tmp/wal/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:35:12,134 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-09-06T20:35:12,136 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:35:12,140 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41733/tmp/wal/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:35:12,140 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10448405600, jitterRate=-0.026916399598121643}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-09-06T20:35:12,140 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-09-06T20:35:12,141 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-09-06T20:35:12,146 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b754f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:12,147 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(860): No meta location available on zookeeper, skip migrating... 2023-09-06T20:35:12,148 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-09-06T20:35:12,148 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(563): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-09-06T20:35:12,148 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-09-06T20:35:12,148 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(583): Recovered RegionProcedureStore lease in 0 msec 2023-09-06T20:35:12,149 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(597): Loaded RegionProcedureStore in 0 msec 2023-09-06T20:35:12,149 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-09-06T20:35:12,151 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-09-06T20:35:12,152 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Unable to get data of znode /2/balancer because node does not exist (not necessarily an error) 2023-09-06T20:35:12,154 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /2/balancer already deleted, retry=false 2023-09-06T20:35:12,154 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(150): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-09-06T20:35:12,154 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Unable to get data of znode /2/normalizer because node does not exist (not necessarily an error) 2023-09-06T20:35:12,156 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /2/normalizer already deleted, retry=false 2023-09-06T20:35:12,157 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-09-06T20:35:12,157 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Unable to get data of znode /2/switch/split because node does not exist (not necessarily an error) 2023-09-06T20:35:12,158 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /2/switch/split already deleted, retry=false 2023-09-06T20:35:12,159 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Unable to get data of znode /2/switch/merge because node does not exist (not necessarily an error) 2023-09-06T20:35:12,160 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /2/switch/merge already deleted, retry=false 2023-09-06T20:35:12,161 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Unable to get data of znode /2/snapshot-cleanup because node does not exist (not necessarily an error) 2023-09-06T20:35:12,163 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /2/snapshot-cleanup already deleted, retry=false 2023-09-06T20:35:12,165 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39115-0x100eefdf7f60005, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/2/running 2023-09-06T20:35:12,165 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/2/running 2023-09-06T20:35:12,165 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-09-06T20:35:12,165 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39115-0x100eefdf7f60005, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-09-06T20:35:12,165 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(804): Active/primary master=jenkins-hbase3.apache.org,41475,1694032510566, sessionid=0x100eefdf7f60004, setting cluster-up flag (Was=false) 2023-09-06T20:35:12,166 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-09-06T20:35:12,168 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39115-0x100eefdf7f60005, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-09-06T20:35:12,168 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-09-06T20:35:12,173 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /2/flush-table-proc/acquired, /2/flush-table-proc/reached, /2/flush-table-proc/abort 2023-09-06T20:35:12,174 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,41475,1694032510566 2023-09-06T20:35:12,180 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-09-06T20:35:12,180 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39115-0x100eefdf7f60005, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-09-06T20:35:12,185 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /2/rolllog-proc/acquired, /2/rolllog-proc/reached, /2/rolllog-proc/abort 2023-09-06T20:35:12,185 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,41475,1694032510566 2023-09-06T20:35:12,189 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-09-06T20:35:12,189 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39115-0x100eefdf7f60005, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-09-06T20:35:12,194 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /2/online-snapshot/acquired, /2/online-snapshot/reached, /2/online-snapshot/abort 2023-09-06T20:35:12,194 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,41475,1694032510566 2023-09-06T20:35:12,198 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-09-06T20:35:12,198 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-09-06T20:35:12,198 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-09-06T20:35:12,199 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,41475,1694032510566 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-09-06T20:35:12,199 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-06T20:35:12,199 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-06T20:35:12,199 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-06T20:35:12,199 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-06T20:35:12,199 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-09-06T20:35:12,199 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:12,199 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-09-06T20:35:12,199 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:12,200 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1694032542200 2023-09-06T20:35:12,201 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-09-06T20:35:12,201 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-09-06T20:35:12,201 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-09-06T20:35:12,201 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-09-06T20:35:12,201 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-09-06T20:35:12,201 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-09-06T20:35:12,201 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-09-06T20:35:12,201 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:12,202 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-09-06T20:35:12,202 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-09-06T20:35:12,202 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-09-06T20:35:12,202 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-09-06T20:35:12,203 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-09-06T20:35:12,204 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x38073e07 to 127.0.0.1:50423 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:35:12,206 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:12,206 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-09-06T20:35:12,208 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a0ee04d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:12,209 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-09-06T20:35:12,209 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-09-06T20:35:12,209 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-09-06T20:35:12,210 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1694032512209,5,FailOnTimeoutGroup] 2023-09-06T20:35:12,210 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1694032512210,5,FailOnTimeoutGroup] 2023-09-06T20:35:12,210 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:12,210 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1635): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-09-06T20:35:12,210 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:12,210 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:12,223 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-09-06T20:35:12,223 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e 2023-09-06T20:35:12,288 DEBUG [RS:0;jenkins-hbase3:39115 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:39115 2023-09-06T20:35:12,289 INFO [RS:0;jenkins-hbase3:39115 {}] regionserver.HRegionServer(997): ClusterId : 839ab22e-31ed-4e78-b1ec-8e29bf999637 2023-09-06T20:35:12,289 INFO [RS:0;jenkins-hbase3:39115 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-09-06T20:35:12,289 DEBUG [RS:0;jenkins-hbase3:39115 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-09-06T20:35:12,292 DEBUG [RS:0;jenkins-hbase3:39115 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-09-06T20:35:12,292 DEBUG [RS:0;jenkins-hbase3:39115 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-09-06T20:35:12,294 DEBUG [RS:0;jenkins-hbase3:39115 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-09-06T20:35:12,294 DEBUG [RS:0;jenkins-hbase3:39115 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-09-06T20:35:12,297 DEBUG [RS:0;jenkins-hbase3:39115 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-09-06T20:35:12,298 DEBUG [RS:0;jenkins-hbase3:39115 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32e24a46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:12,299 DEBUG [RS:0;jenkins-hbase3:39115 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dcad5d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-09-06T20:35:12,299 INFO [RS:0;jenkins-hbase3:39115 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-09-06T20:35:12,299 INFO [RS:0;jenkins-hbase3:39115 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-09-06T20:35:12,299 DEBUG [RS:0;jenkins-hbase3:39115 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-09-06T20:35:12,300 INFO [RS:0;jenkins-hbase3:39115 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,41475,1694032510566 with isa=jenkins-hbase3.apache.org/172.31.12.81:39115, startcode=1694032510720 2023-09-06T20:35:12,300 DEBUG [RS:0;jenkins-hbase3:39115 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-09-06T20:35:12,303 INFO [RS-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:52733, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2023-09-06T20:35:12,304 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41475 {}] master.ServerManager(403): Registering regionserver=jenkins-hbase3.apache.org,39115,1694032510720 2023-09-06T20:35:12,306 DEBUG [RS:0;jenkins-hbase3:39115 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e 2023-09-06T20:35:12,306 DEBUG [RS:0;jenkins-hbase3:39115 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:41733 2023-09-06T20:35:12,306 DEBUG [RS:0;jenkins-hbase3:39115 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=45621 2023-09-06T20:35:12,308 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/rs 2023-09-06T20:35:12,309 DEBUG [RS:0;jenkins-hbase3:39115 {}] zookeeper.ZKUtil(111): regionserver:39115-0x100eefdf7f60005, quorum=127.0.0.1:50423, baseZNode=/2 Set watcher on existing znode=/2/rs/jenkins-hbase3.apache.org,39115,1694032510720 2023-09-06T20:35:12,309 WARN [RS:0;jenkins-hbase3:39115 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-09-06T20:35:12,309 INFO [RS:0;jenkins-hbase3:39115 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-09-06T20:35:12,309 DEBUG [RS:0;jenkins-hbase3:39115 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:41733/tmp/wal/WALs/jenkins-hbase3.apache.org,39115,1694032510720 2023-09-06T20:35:12,309 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,39115,1694032510720] 2023-09-06T20:35:12,314 DEBUG [RS:0;jenkins-hbase3:39115 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-09-06T20:35:12,314 INFO [RS:0;jenkins-hbase3:39115 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-09-06T20:35:12,316 INFO [RS:0;jenkins-hbase3:39115 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-09-06T20:35:12,318 INFO [RS:0;jenkins-hbase3:39115 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-09-06T20:35:12,318 INFO [RS:0;jenkins-hbase3:39115 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:12,320 INFO [RS:0;jenkins-hbase3:39115 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-09-06T20:35:12,321 INFO [RS:0;jenkins-hbase3:39115 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:12,321 DEBUG [RS:0;jenkins-hbase3:39115 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:12,322 DEBUG [RS:0;jenkins-hbase3:39115 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:12,322 DEBUG [RS:0;jenkins-hbase3:39115 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:12,322 DEBUG [RS:0;jenkins-hbase3:39115 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:12,322 DEBUG [RS:0;jenkins-hbase3:39115 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:12,322 DEBUG [RS:0;jenkins-hbase3:39115 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-09-06T20:35:12,322 DEBUG [RS:0;jenkins-hbase3:39115 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:12,322 DEBUG [RS:0;jenkins-hbase3:39115 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:12,322 DEBUG [RS:0;jenkins-hbase3:39115 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:12,322 DEBUG [RS:0;jenkins-hbase3:39115 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:12,322 DEBUG [RS:0;jenkins-hbase3:39115 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:35:12,322 DEBUG [RS:0;jenkins-hbase3:39115 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-09-06T20:35:12,322 DEBUG [RS:0;jenkins-hbase3:39115 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-09-06T20:35:12,323 INFO [RS:0;jenkins-hbase3:39115 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:12,323 INFO [RS:0;jenkins-hbase3:39115 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:12,323 INFO [RS:0;jenkins-hbase3:39115 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:12,323 INFO [RS:0;jenkins-hbase3:39115 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:12,323 INFO [RS:0;jenkins-hbase3:39115 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,39115,1694032510720-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-09-06T20:35:12,334 INFO [RS:0;jenkins-hbase3:39115 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-09-06T20:35:12,335 INFO [RS:0;jenkins-hbase3:39115 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,39115,1694032510720-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:12,346 INFO [RS:0;jenkins-hbase3:39115 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,39115,1694032510720 started 2023-09-06T20:35:12,346 INFO [RS:0;jenkins-hbase3:39115 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,39115,1694032510720, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:39115, sessionid=0x100eefdf7f60005 2023-09-06T20:35:12,346 DEBUG [RS:0;jenkins-hbase3:39115 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-09-06T20:35:12,346 DEBUG [RS:0;jenkins-hbase3:39115 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager jenkins-hbase3.apache.org,39115,1694032510720 2023-09-06T20:35:12,346 DEBUG [RS:0;jenkins-hbase3:39115 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,39115,1694032510720' 2023-09-06T20:35:12,346 DEBUG [RS:0;jenkins-hbase3:39115 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/2/flush-table-proc/abort' 2023-09-06T20:35:12,346 DEBUG [RS:0;jenkins-hbase3:39115 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/2/flush-table-proc/acquired' 2023-09-06T20:35:12,347 DEBUG [RS:0;jenkins-hbase3:39115 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-09-06T20:35:12,347 DEBUG [RS:0;jenkins-hbase3:39115 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-09-06T20:35:12,347 DEBUG [RS:0;jenkins-hbase3:39115 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,39115,1694032510720' 2023-09-06T20:35:12,347 DEBUG [RS:0;jenkins-hbase3:39115 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/2/rolllog-proc/abort' 2023-09-06T20:35:12,347 DEBUG [RS:0;jenkins-hbase3:39115 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/2/rolllog-proc/acquired' 2023-09-06T20:35:12,348 INFO [RS:0;jenkins-hbase3:39115 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-09-06T20:35:12,348 DEBUG [RS:0;jenkins-hbase3:39115 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-09-06T20:35:12,348 DEBUG [RS:0;jenkins-hbase3:39115 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-09-06T20:35:12,348 DEBUG [RS:0;jenkins-hbase3:39115 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,39115,1694032510720 2023-09-06T20:35:12,348 DEBUG [RS:0;jenkins-hbase3:39115 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,39115,1694032510720' 2023-09-06T20:35:12,348 DEBUG [RS:0;jenkins-hbase3:39115 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/2/online-snapshot/abort' 2023-09-06T20:35:12,348 DEBUG [RS:0;jenkins-hbase3:39115 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/2/online-snapshot/acquired' 2023-09-06T20:35:12,349 DEBUG [RS:0;jenkins-hbase3:39115 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-09-06T20:35:12,349 INFO [RS:0;jenkins-hbase3:39115 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-09-06T20:35:12,349 INFO [RS:0;jenkins-hbase3:39115 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-09-06T20:35:12,449 INFO [RS:0;jenkins-hbase3:39115 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-09-06T20:35:12,451 INFO [RS:0;jenkins-hbase3:39115 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C39115%2C1694032510720, suffix=, logDir=hdfs://localhost:41733/tmp/wal/WALs/jenkins-hbase3.apache.org,39115,1694032510720, archiveDir=hdfs://localhost:41733/tmp/wal/oldWALs, maxLogs=32 2023-09-06T20:35:12,463 DEBUG [RS:0;jenkins-hbase3:39115 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /tmp/wal/WALs/jenkins-hbase3.apache.org,39115,1694032510720/jenkins-hbase3.apache.org%2C39115%2C1694032510720.1694032512452, exclude list is [], retry=0 2023-09-06T20:35:12,468 DEBUG [RS-EventLoopGroup-6-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38883,DS-71098be5-7b27-4f7d-ab7a-e0614e889cd8,DISK] 2023-09-06T20:35:12,471 INFO [RS:0;jenkins-hbase3:39115 {}] wal.AbstractFSWAL(832): New WAL /tmp/wal/WALs/jenkins-hbase3.apache.org,39115,1694032510720/jenkins-hbase3.apache.org%2C39115%2C1694032510720.1694032512452 2023-09-06T20:35:12,472 DEBUG [RS:0;jenkins-hbase3:39115 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38883,DS-71098be5-7b27-4f7d-ab7a-e0614e889cd8,DISK]] 2023-09-06T20:35:12,640 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:12,642 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-09-06T20:35:12,644 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-09-06T20:35:12,645 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:12,645 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:35:12,646 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-09-06T20:35:12,647 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-09-06T20:35:12,647 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:12,648 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:35:12,648 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-09-06T20:35:12,650 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-09-06T20:35:12,650 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:12,650 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:35:12,652 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/data/hbase/meta/1588230740 2023-09-06T20:35:12,652 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/data/hbase/meta/1588230740 2023-09-06T20:35:12,653 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41733/tmp/wal/data/hbase/meta/1588230740 2023-09-06T20:35:12,655 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-09-06T20:35:12,657 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-09-06T20:35:12,660 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41733/tmp/wal/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:35:12,661 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10778300480, jitterRate=0.00380745530128479}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-09-06T20:35:12,662 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-09-06T20:35:12,662 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-09-06T20:35:12,662 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-09-06T20:35:12,662 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-09-06T20:35:12,662 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-09-06T20:35:12,662 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-09-06T20:35:12,662 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-09-06T20:35:12,662 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-09-06T20:35:12,664 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-09-06T20:35:12,664 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-09-06T20:35:12,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-09-06T20:35:12,666 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-09-06T20:35:12,668 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-09-06T20:35:12,784 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2023-09-06T20:35:12,819 DEBUG [jenkins-hbase3:41475 {}] assignment.AssignmentManager(2303): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-09-06T20:35:12,826 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,39115,1694032510720 2023-09-06T20:35:12,830 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,39115,1694032510720, state=OPENING 2023-09-06T20:35:12,831 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-09-06T20:35:12,833 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39115-0x100eefdf7f60005, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-09-06T20:35:12,833 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-09-06T20:35:12,833 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /2/meta-region-server: CHANGED 2023-09-06T20:35:12,834 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /2/meta-region-server: CHANGED 2023-09-06T20:35:12,834 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,39115,1694032510720}] 2023-09-06T20:35:12,871 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-09-06T20:35:12,873 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-09-06T20:35:12,875 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-09-06T20:35:12,875 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-09-06T20:35:12,877 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-09-06T20:35:12,877 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-09-06T20:35:12,878 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:35:12,878 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2023-09-06T20:35:12,878 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-09-06T20:35:12,878 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-09-06T20:35:12,878 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-09-06T20:35:12,878 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-09-06T20:35:12,991 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(721): New admin connection to jenkins-hbase3.apache.org,39115,1694032510720 2023-09-06T20:35:12,992 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2023-09-06T20:35:12,994 INFO [RS-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:60644, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-09-06T20:35:13,001 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-09-06T20:35:13,001 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-09-06T20:35:13,001 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-09-06T20:35:13,003 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C39115%2C1694032510720.meta, suffix=.meta, logDir=hdfs://localhost:41733/tmp/wal/WALs/jenkins-hbase3.apache.org,39115,1694032510720, archiveDir=hdfs://localhost:41733/tmp/wal/oldWALs, maxLogs=32 2023-09-06T20:35:13,014 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /tmp/wal/WALs/jenkins-hbase3.apache.org,39115,1694032510720/jenkins-hbase3.apache.org%2C39115%2C1694032510720.meta.1694032513004.meta, exclude list is [], retry=0 2023-09-06T20:35:13,018 DEBUG [RS-EventLoopGroup-6-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46471,DS-5c3dda06-912e-4de1-b561-0f5c1ff9ef54,DISK] 2023-09-06T20:35:13,021 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /tmp/wal/WALs/jenkins-hbase3.apache.org,39115,1694032510720/jenkins-hbase3.apache.org%2C39115%2C1694032510720.meta.1694032513004.meta 2023-09-06T20:35:13,021 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:46471,DS-5c3dda06-912e-4de1-b561-0f5c1ff9ef54,DISK]] 2023-09-06T20:35:13,021 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:35:13,022 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:35:13,022 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-09-06T20:35:13,022 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-09-06T20:35:13,022 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-09-06T20:35:13,022 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-09-06T20:35:13,022 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:13,022 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-09-06T20:35:13,022 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-09-06T20:35:13,024 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-09-06T20:35:13,025 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-09-06T20:35:13,025 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:13,026 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:35:13,026 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-09-06T20:35:13,027 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-09-06T20:35:13,027 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:13,027 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:35:13,028 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-09-06T20:35:13,029 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-09-06T20:35:13,029 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:13,029 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:35:13,030 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/data/hbase/meta/1588230740 2023-09-06T20:35:13,031 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/data/hbase/meta/1588230740 2023-09-06T20:35:13,032 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41733/tmp/wal/data/hbase/meta/1588230740 2023-09-06T20:35:13,036 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-09-06T20:35:13,038 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-09-06T20:35:13,039 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=12078270880, jitterRate=0.12487663328647614}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-09-06T20:35:13,039 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-09-06T20:35:13,040 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1694032512991 2023-09-06T20:35:13,045 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:meta,,1.1588230740 2023-09-06T20:35:13,045 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-09-06T20:35:13,046 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,39115,1694032510720 2023-09-06T20:35:13,047 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,39115,1694032510720, state=OPEN 2023-09-06T20:35:13,050 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/2/meta-region-server 2023-09-06T20:35:13,050 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39115-0x100eefdf7f60005, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/2/meta-region-server 2023-09-06T20:35:13,050 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /2/meta-region-server: CHANGED 2023-09-06T20:35:13,050 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /2/meta-region-server: CHANGED 2023-09-06T20:35:13,053 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=3, resume processing ppid=2 2023-09-06T20:35:13,053 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,39115,1694032510720 in 217 msec 2023-09-06T20:35:13,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=2, resume processing ppid=1 2023-09-06T20:35:13,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 389 msec 2023-09-06T20:35:13,060 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 862 msec 2023-09-06T20:35:13,060 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1063): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1694032513060, completionTime=-1 2023-09-06T20:35:13,060 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(830): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-09-06T20:35:13,060 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1629): Joining cluster... 2023-09-06T20:35:13,062 DEBUG [hconnection-0x50d7c39e-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:13,064 INFO [RS-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:60650, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:13,065 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1641): Number of RegionServers=1 2023-09-06T20:35:13,065 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1694032573065 2023-09-06T20:35:13,066 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1694032633066 2023-09-06T20:35:13,066 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1648): Joined the cluster in 5 msec 2023-09-06T20:35:13,073 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,41475,1694032510566-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:13,073 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,41475,1694032510566-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:13,073 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,41475,1694032510566-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:13,073 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:41475, period=300000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:13,073 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-09-06T20:35:13,073 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-09-06T20:35:13,073 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2368): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-09-06T20:35:13,075 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-09-06T20:35:13,075 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-09-06T20:35:13,076 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:35:13,077 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:13,078 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:35:13,496 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 932b31f1be4976500c94245ad49c9242, NAME => 'hbase:namespace,,1694032513073.932b31f1be4976500c94245ad49c9242.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e 2023-09-06T20:35:13,508 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1694032513073.932b31f1be4976500c94245ad49c9242.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:13,508 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing 932b31f1be4976500c94245ad49c9242, disabling compactions & flushes 2023-09-06T20:35:13,508 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1694032513073.932b31f1be4976500c94245ad49c9242. 2023-09-06T20:35:13,508 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1694032513073.932b31f1be4976500c94245ad49c9242. 2023-09-06T20:35:13,508 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1694032513073.932b31f1be4976500c94245ad49c9242. after waiting 0 ms 2023-09-06T20:35:13,508 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1694032513073.932b31f1be4976500c94245ad49c9242. 2023-09-06T20:35:13,508 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1694032513073.932b31f1be4976500c94245ad49c9242. 2023-09-06T20:35:13,508 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for 932b31f1be4976500c94245ad49c9242: 2023-09-06T20:35:13,510 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:35:13,511 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1694032513073.932b31f1be4976500c94245ad49c9242.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1694032513510"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032513510"}]},"ts":"1694032513510"} 2023-09-06T20:35:13,514 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:35:13,515 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:35:13,516 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032513515"}]},"ts":"1694032513515"} 2023-09-06T20:35:13,518 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-09-06T20:35:13,528 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=932b31f1be4976500c94245ad49c9242, ASSIGN}] 2023-09-06T20:35:13,530 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=932b31f1be4976500c94245ad49c9242, ASSIGN 2023-09-06T20:35:13,531 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=932b31f1be4976500c94245ad49c9242, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,39115,1694032510720; forceNewPlan=false, retain=false 2023-09-06T20:35:13,682 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=932b31f1be4976500c94245ad49c9242, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,39115,1694032510720 2023-09-06T20:35:13,685 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 932b31f1be4976500c94245ad49c9242, server=jenkins-hbase3.apache.org,39115,1694032510720}] 2023-09-06T20:35:13,845 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1694032513073.932b31f1be4976500c94245ad49c9242. 2023-09-06T20:35:13,845 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => 932b31f1be4976500c94245ad49c9242, NAME => 'hbase:namespace,,1694032513073.932b31f1be4976500c94245ad49c9242.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:35:13,846 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:35:13,846 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 932b31f1be4976500c94245ad49c9242 2023-09-06T20:35:13,846 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1694032513073.932b31f1be4976500c94245ad49c9242.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:13,846 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for 932b31f1be4976500c94245ad49c9242 2023-09-06T20:35:13,846 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for 932b31f1be4976500c94245ad49c9242 2023-09-06T20:35:13,848 INFO [StoreOpener-932b31f1be4976500c94245ad49c9242-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 932b31f1be4976500c94245ad49c9242 2023-09-06T20:35:13,850 INFO [StoreOpener-932b31f1be4976500c94245ad49c9242-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 932b31f1be4976500c94245ad49c9242 columnFamilyName info 2023-09-06T20:35:13,850 DEBUG [StoreOpener-932b31f1be4976500c94245ad49c9242-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:13,851 INFO [StoreOpener-932b31f1be4976500c94245ad49c9242-1 {}] regionserver.HStore(324): Store=932b31f1be4976500c94245ad49c9242/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:35:13,852 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/data/hbase/namespace/932b31f1be4976500c94245ad49c9242 2023-09-06T20:35:13,853 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/data/hbase/namespace/932b31f1be4976500c94245ad49c9242 2023-09-06T20:35:13,854 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41733/tmp/wal/data/hbase/namespace/932b31f1be4976500c94245ad49c9242 2023-09-06T20:35:13,857 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for 932b31f1be4976500c94245ad49c9242 2023-09-06T20:35:13,860 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41733/tmp/wal/data/hbase/namespace/932b31f1be4976500c94245ad49c9242/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:35:13,860 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened 932b31f1be4976500c94245ad49c9242; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11652287360, jitterRate=0.08520382642745972}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:35:13,860 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for 932b31f1be4976500c94245ad49c9242: 2023-09-06T20:35:13,862 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:namespace,,1694032513073.932b31f1be4976500c94245ad49c9242., pid=6, masterSystemTime=1694032513838 2023-09-06T20:35:13,865 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:namespace,,1694032513073.932b31f1be4976500c94245ad49c9242. 2023-09-06T20:35:13,865 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1694032513073.932b31f1be4976500c94245ad49c9242. 2023-09-06T20:35:13,866 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=932b31f1be4976500c94245ad49c9242, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,39115,1694032510720 2023-09-06T20:35:13,872 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=6, resume processing ppid=5 2023-09-06T20:35:13,872 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 932b31f1be4976500c94245ad49c9242, server=jenkins-hbase3.apache.org,39115,1694032510720 in 184 msec 2023-09-06T20:35:13,875 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=5, resume processing ppid=4 2023-09-06T20:35:13,875 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=932b31f1be4976500c94245ad49c9242, ASSIGN in 344 msec 2023-09-06T20:35:13,876 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:35:13,877 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032513876"}]},"ts":"1694032513876"} 2023-09-06T20:35:13,879 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-09-06T20:35:13,882 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:35:13,885 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 809 msec 2023-09-06T20:35:13,976 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Set watcher on znode that does not yet exist, /2/namespace 2023-09-06T20:35:13,978 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39115-0x100eefdf7f60005, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-09-06T20:35:13,978 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/2/namespace 2023-09-06T20:35:13,978 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-09-06T20:35:13,984 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-09-06T20:35:13,995 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/namespace 2023-09-06T20:35:14,001 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 16 msec 2023-09-06T20:35:14,006 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-09-06T20:35:14,016 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/namespace 2023-09-06T20:35:14,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 12 msec 2023-09-06T20:35:14,032 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/2/namespace/default 2023-09-06T20:35:14,034 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/2/namespace/hbase 2023-09-06T20:35:14,034 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1193): Master has completed initialization 3.244sec 2023-09-06T20:35:14,035 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-09-06T20:35:14,035 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-09-06T20:35:14,035 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-09-06T20:35:14,035 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-09-06T20:35:14,035 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-09-06T20:35:14,035 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,41475,1694032510566-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-09-06T20:35:14,035 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,41475,1694032510566-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-09-06T20:35:14,037 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1296): Balancer post startup initialization complete, took 0 seconds 2023-09-06T20:35:14,037 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-09-06T20:35:14,074 DEBUG [Listener at localhost/39521 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5b70bf78 to 127.0.0.1:50423 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:35:14,080 DEBUG [Listener at localhost/39521 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4df6b849, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:14,081 DEBUG [hconnection-0x6acd9a07-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:14,084 INFO [RS-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:60656, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:14,086 INFO [Listener at localhost/39521 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,41475,1694032510566 2023-09-06T20:35:14,086 INFO [Listener at localhost/39521 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-09-06T20:35:14,086 INFO [Listener at localhost/39521 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/test.cache.data in system properties and HBase conf 2023-09-06T20:35:14,086 INFO [Listener at localhost/39521 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/hadoop.tmp.dir in system properties and HBase conf 2023-09-06T20:35:14,086 INFO [Listener at localhost/39521 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/hadoop.log.dir in system properties and HBase conf 2023-09-06T20:35:14,086 INFO [Listener at localhost/39521 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/mapreduce.cluster.local.dir in system properties and HBase conf 2023-09-06T20:35:14,086 INFO [Listener at localhost/39521 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-09-06T20:35:14,086 INFO [Listener at localhost/39521 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-09-06T20:35:14,086 INFO [Listener at localhost/39521 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-09-06T20:35:14,087 INFO [Listener at localhost/39521 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-09-06T20:35:14,087 INFO [Listener at localhost/39521 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-09-06T20:35:14,087 INFO [Listener at localhost/39521 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-09-06T20:35:14,087 INFO [Listener at localhost/39521 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-09-06T20:35:14,087 INFO [Listener at localhost/39521 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-09-06T20:35:14,087 INFO [Listener at localhost/39521 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-09-06T20:35:14,087 INFO [Listener at localhost/39521 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/dfs.journalnode.edits.dir in system properties and HBase conf 2023-09-06T20:35:14,087 INFO [Listener at localhost/39521 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-09-06T20:35:14,087 INFO [Listener at localhost/39521 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/nfs.dump.dir in system properties and HBase conf 2023-09-06T20:35:14,087 INFO [Listener at localhost/39521 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/java.io.tmpdir in system properties and HBase conf 2023-09-06T20:35:14,087 INFO [Listener at localhost/39521 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/dfs.journalnode.edits.dir in system properties and HBase conf 2023-09-06T20:35:14,087 INFO [Listener at localhost/39521 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-09-06T20:35:14,087 INFO [Listener at localhost/39521 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-09-06T20:35:15,178 WARN [Thread-467 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:35:15,540 INFO [Thread-467 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:35:15,546 INFO [Thread-467 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_46501_jobhistory____.hib9mq/webapp 2023-09-06T20:35:15,711 INFO [Thread-467 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-09-06T20:35:15,711 INFO [Thread-467 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-09-06T20:35:15,711 INFO [Thread-467 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-09-06T20:35:15,716 INFO [Thread-467 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-09-06T20:35:15,816 INFO [Thread-467 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:35:15,932 INFO [Thread-467 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:35:16,226 INFO [Thread-467 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-09-06T20:35:16,266 INFO [Thread-467 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:46501 2023-09-06T20:35:16,674 WARN [Listener at jenkins-hbase3.apache.org/46789 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:35:16,763 WARN [Listener at jenkins-hbase3.apache.org/46789 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-09-06T20:35:16,764 INFO [Listener at jenkins-hbase3.apache.org/46789 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:35:16,776 INFO [Listener at jenkins-hbase3.apache.org/46789 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_44945_cluster____.svi6fr/webapp 2023-09-06T20:35:16,854 INFO [Listener at jenkins-hbase3.apache.org/46789 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-09-06T20:35:16,854 INFO [Listener at jenkins-hbase3.apache.org/46789 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-09-06T20:35:16,854 INFO [Listener at jenkins-hbase3.apache.org/46789 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-09-06T20:35:16,855 INFO [Listener at jenkins-hbase3.apache.org/46789 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-09-06T20:35:16,863 INFO [Listener at jenkins-hbase3.apache.org/46789 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:35:17,051 INFO [Listener at jenkins-hbase3.apache.org/46789 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:35:17,157 INFO [Listener at jenkins-hbase3.apache.org/46789 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:35:17,165 INFO [Listener at jenkins-hbase3.apache.org/46789 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:44945 2023-09-06T20:35:17,502 WARN [Listener at jenkins-hbase3.apache.org/44359 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:35:17,528 WARN [Listener at jenkins-hbase3.apache.org/44359 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-09-06T20:35:17,529 INFO [Listener at jenkins-hbase3.apache.org/44359 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:35:17,538 INFO [Listener at jenkins-hbase3.apache.org/44359 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_42581_node____.btglrl/webapp 2023-09-06T20:35:17,592 INFO [Listener at jenkins-hbase3.apache.org/44359 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-09-06T20:35:17,592 INFO [Listener at jenkins-hbase3.apache.org/44359 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-09-06T20:35:17,592 INFO [Listener at jenkins-hbase3.apache.org/44359 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-09-06T20:35:17,593 INFO [Listener at jenkins-hbase3.apache.org/44359 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-09-06T20:35:17,601 INFO [Listener at jenkins-hbase3.apache.org/44359 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:35:17,627 INFO [Listener at jenkins-hbase3.apache.org/44359 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:35:17,709 INFO [Listener at jenkins-hbase3.apache.org/44359 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:35:17,720 INFO [Listener at jenkins-hbase3.apache.org/44359 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:42581 2023-09-06T20:35:17,882 WARN [Listener at jenkins-hbase3.apache.org/34023 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:35:17,888 WARN [Listener at jenkins-hbase3.apache.org/34023 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-09-06T20:35:17,889 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:35:17,900 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_39279_node____3b3hjh/webapp 2023-09-06T20:35:17,958 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-09-06T20:35:17,959 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-09-06T20:35:17,959 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-09-06T20:35:17,959 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-09-06T20:35:17,970 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:35:18,061 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:35:18,134 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:35:18,143 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:39279 2023-09-06T20:35:18,178 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-09-06T20:35:18,179 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] backup.TestBackupBase(301): ROOTDIR hdfs://localhost:36869/backupUT 2023-09-06T20:35:18,189 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] backup.TestBackupBase(306): REMOTE ROOTDIR hdfs://localhost:41733/backupUT 2023-09-06T20:35:18,202 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:35:18,205 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54800, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:35:18,216 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns1'} 2023-09-06T20:35:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] procedure2.ProcedureExecutor(1032): Stored pid=9, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns1 2023-09-06T20:35:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-09-06T20:35:18,240 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:35:18,245 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=9, state=SUCCESS; CreateNamespaceProcedure, namespace=ns1 in 26 msec 2023-09-06T20:35:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-09-06T20:35:18,337 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns2'} 2023-09-06T20:35:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] procedure2.ProcedureExecutor(1032): Stored pid=10, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns2 2023-09-06T20:35:18,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=10 2023-09-06T20:35:18,350 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:35:18,357 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=10, state=SUCCESS; CreateNamespaceProcedure, namespace=ns2 in 18 msec 2023-09-06T20:35:18,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=10 2023-09-06T20:35:18,448 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns3'} 2023-09-06T20:35:18,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] procedure2.ProcedureExecutor(1032): Stored pid=11, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns3 2023-09-06T20:35:18,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=11 2023-09-06T20:35:18,460 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:35:18,465 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=11, state=SUCCESS; CreateNamespaceProcedure, namespace=ns3 in 15 msec 2023-09-06T20:35:18,550 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:35:18,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=11 2023-09-06T20:35:18,564 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns4'} 2023-09-06T20:35:18,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] procedure2.ProcedureExecutor(1032): Stored pid=12, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns4 2023-09-06T20:35:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-09-06T20:35:18,590 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:35:18,608 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=12, state=SUCCESS; CreateNamespaceProcedure, namespace=ns4 in 43 msec 2023-09-06T20:35:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-09-06T20:35:18,699 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'test-1694032518189', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:35:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] procedure2.ProcedureExecutor(1032): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=test-1694032518189 2023-09-06T20:35:18,704 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=test-1694032518189 execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:35:18,704 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "test-1694032518189" procId is: 13 2023-09-06T20:35:18,705 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:18,706 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=test-1694032518189 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:35:18,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-09-06T20:35:18,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-09-06T20:35:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-09-06T20:35:19,131 INFO [RegionOpenAndInit-test-1694032518189-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => c5da9ae4b22d80dadf15a6cb7c09b7e4, NAME => 'test-1694032518189,,1694032518692.c5da9ae4b22d80dadf15a6cb7c09b7e4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1694032518189', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd 2023-09-06T20:35:19,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-09-06T20:35:19,545 DEBUG [RegionOpenAndInit-test-1694032518189-pool-0 {}] regionserver.HRegion(888): Instantiated test-1694032518189,,1694032518692.c5da9ae4b22d80dadf15a6cb7c09b7e4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:19,545 DEBUG [RegionOpenAndInit-test-1694032518189-pool-0 {}] regionserver.HRegion(1668): Closing c5da9ae4b22d80dadf15a6cb7c09b7e4, disabling compactions & flushes 2023-09-06T20:35:19,545 INFO [RegionOpenAndInit-test-1694032518189-pool-0 {}] regionserver.HRegion(1690): Closing region test-1694032518189,,1694032518692.c5da9ae4b22d80dadf15a6cb7c09b7e4. 2023-09-06T20:35:19,545 DEBUG [RegionOpenAndInit-test-1694032518189-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1694032518189,,1694032518692.c5da9ae4b22d80dadf15a6cb7c09b7e4. 2023-09-06T20:35:19,545 DEBUG [RegionOpenAndInit-test-1694032518189-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1694032518189,,1694032518692.c5da9ae4b22d80dadf15a6cb7c09b7e4. after waiting 0 ms 2023-09-06T20:35:19,545 DEBUG [RegionOpenAndInit-test-1694032518189-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1694032518189,,1694032518692.c5da9ae4b22d80dadf15a6cb7c09b7e4. 2023-09-06T20:35:19,545 INFO [RegionOpenAndInit-test-1694032518189-pool-0 {}] regionserver.HRegion(1909): Closed test-1694032518189,,1694032518692.c5da9ae4b22d80dadf15a6cb7c09b7e4. 2023-09-06T20:35:19,545 DEBUG [RegionOpenAndInit-test-1694032518189-pool-0 {}] regionserver.HRegion(1622): Region close journal for c5da9ae4b22d80dadf15a6cb7c09b7e4: 2023-09-06T20:35:19,547 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=test-1694032518189 execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:35:19,547 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1694032518189,,1694032518692.c5da9ae4b22d80dadf15a6cb7c09b7e4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1694032519547"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032519547"}]},"ts":"1694032519547"} 2023-09-06T20:35:19,549 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:35:19,551 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=test-1694032518189 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:35:19,551 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1694032518189","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032519551"}]},"ts":"1694032519551"} 2023-09-06T20:35:19,553 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1694032518189, state=ENABLING in hbase:meta 2023-09-06T20:35:19,556 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1694032518189, region=c5da9ae4b22d80dadf15a6cb7c09b7e4, ASSIGN}] 2023-09-06T20:35:19,559 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1694032518189, region=c5da9ae4b22d80dadf15a6cb7c09b7e4, ASSIGN 2023-09-06T20:35:19,560 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1694032518189, region=c5da9ae4b22d80dadf15a6cb7c09b7e4, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36075,1694032504561; forceNewPlan=false, retain=false 2023-09-06T20:35:19,712 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=c5da9ae4b22d80dadf15a6cb7c09b7e4, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:19,715 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; OpenRegionProcedure c5da9ae4b22d80dadf15a6cb7c09b7e4, server=jenkins-hbase3.apache.org,36075,1694032504561}] 2023-09-06T20:35:19,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-09-06T20:35:19,879 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(130): Open test-1694032518189,,1694032518692.c5da9ae4b22d80dadf15a6cb7c09b7e4. 2023-09-06T20:35:19,879 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7254): Opening region: {ENCODED => c5da9ae4b22d80dadf15a6cb7c09b7e4, NAME => 'test-1694032518189,,1694032518692.c5da9ae4b22d80dadf15a6cb7c09b7e4.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:35:19,879 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:35:19,879 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1694032518189 c5da9ae4b22d80dadf15a6cb7c09b7e4 2023-09-06T20:35:19,879 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(888): Instantiated test-1694032518189,,1694032518692.c5da9ae4b22d80dadf15a6cb7c09b7e4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:19,879 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7296): checking encryption for c5da9ae4b22d80dadf15a6cb7c09b7e4 2023-09-06T20:35:19,880 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7299): checking classloading for c5da9ae4b22d80dadf15a6cb7c09b7e4 2023-09-06T20:35:19,882 INFO [StoreOpener-c5da9ae4b22d80dadf15a6cb7c09b7e4-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region c5da9ae4b22d80dadf15a6cb7c09b7e4 2023-09-06T20:35:19,884 INFO [StoreOpener-c5da9ae4b22d80dadf15a6cb7c09b7e4-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5da9ae4b22d80dadf15a6cb7c09b7e4 columnFamilyName f 2023-09-06T20:35:19,884 DEBUG [StoreOpener-c5da9ae4b22d80dadf15a6cb7c09b7e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:19,885 INFO [StoreOpener-c5da9ae4b22d80dadf15a6cb7c09b7e4-1 {}] regionserver.HStore(324): Store=c5da9ae4b22d80dadf15a6cb7c09b7e4/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:35:19,886 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/test-1694032518189/c5da9ae4b22d80dadf15a6cb7c09b7e4 2023-09-06T20:35:19,886 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/test-1694032518189/c5da9ae4b22d80dadf15a6cb7c09b7e4 2023-09-06T20:35:19,889 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1076): writing seq id for c5da9ae4b22d80dadf15a6cb7c09b7e4 2023-09-06T20:35:19,898 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/test-1694032518189/c5da9ae4b22d80dadf15a6cb7c09b7e4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:35:19,899 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): Opened c5da9ae4b22d80dadf15a6cb7c09b7e4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11422505120, jitterRate=0.06380368769168854}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:35:19,900 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(992): Region open journal for c5da9ae4b22d80dadf15a6cb7c09b7e4: 2023-09-06T20:35:19,901 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2556): Post open deploy tasks for test-1694032518189,,1694032518692.c5da9ae4b22d80dadf15a6cb7c09b7e4., pid=15, masterSystemTime=1694032519868 2023-09-06T20:35:19,904 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2583): Finished post open deploy task for test-1694032518189,,1694032518692.c5da9ae4b22d80dadf15a6cb7c09b7e4. 2023-09-06T20:35:19,904 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(158): Opened test-1694032518189,,1694032518692.c5da9ae4b22d80dadf15a6cb7c09b7e4. 2023-09-06T20:35:19,905 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=c5da9ae4b22d80dadf15a6cb7c09b7e4, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:19,910 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=15, resume processing ppid=14 2023-09-06T20:35:19,910 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=15, ppid=14, state=SUCCESS; OpenRegionProcedure c5da9ae4b22d80dadf15a6cb7c09b7e4, server=jenkins-hbase3.apache.org,36075,1694032504561 in 192 msec 2023-09-06T20:35:19,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=14, resume processing ppid=13 2023-09-06T20:35:19,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=14, ppid=13, state=SUCCESS; TransitRegionStateProcedure table=test-1694032518189, region=c5da9ae4b22d80dadf15a6cb7c09b7e4, ASSIGN in 354 msec 2023-09-06T20:35:19,914 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=test-1694032518189 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:35:19,914 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1694032518189","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032519914"}]},"ts":"1694032519914"} 2023-09-06T20:35:19,916 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1694032518189, state=ENABLED in hbase:meta 2023-09-06T20:35:19,919 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=test-1694032518189 execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:35:19,921 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=13, state=SUCCESS; CreateTableProcedure table=test-1694032518189 in 1.2200 sec 2023-09-06T20:35:20,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-09-06T20:35:20,820 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:test-1694032518189, procId: 13 completed 2023-09-06T20:35:20,823 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1191cd9f to 127.0.0.1:50423 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:35:20,833 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@213ce3cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:20,837 DEBUG [hconnection-0x582e99c3-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:20,842 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:60800, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:20,853 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] regionserver.HRegion(8210): writing data to region test-1694032518189,,1694032518692.c5da9ae4b22d80dadf15a6cb7c09b7e4. with WAL disabled. Data may be lost in the event of a crash. 2023-09-06T20:35:20,952 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns2:test-16940325181891', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:35:20,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] procedure2.ProcedureExecutor(1032): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:test-16940325181891 2023-09-06T20:35:20,957 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16940325181891 execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:35:20,957 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:20,957 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "test-16940325181891" procId is: 16 2023-09-06T20:35:20,958 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:test-16940325181891 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:35:20,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-06T20:35:21,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-06T20:35:21,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-06T20:35:21,376 INFO [RegionOpenAndInit-ns2:test-16940325181891-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => db7e5fe8ab24349786ccf3415858edef, NAME => 'ns2:test-16940325181891,,1694032520952.db7e5fe8ab24349786ccf3415858edef.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16940325181891', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd 2023-09-06T20:35:21,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-06T20:35:21,791 DEBUG [RegionOpenAndInit-ns2:test-16940325181891-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16940325181891,,1694032520952.db7e5fe8ab24349786ccf3415858edef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:21,791 DEBUG [RegionOpenAndInit-ns2:test-16940325181891-pool-0 {}] regionserver.HRegion(1668): Closing db7e5fe8ab24349786ccf3415858edef, disabling compactions & flushes 2023-09-06T20:35:21,791 INFO [RegionOpenAndInit-ns2:test-16940325181891-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16940325181891,,1694032520952.db7e5fe8ab24349786ccf3415858edef. 2023-09-06T20:35:21,791 DEBUG [RegionOpenAndInit-ns2:test-16940325181891-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16940325181891,,1694032520952.db7e5fe8ab24349786ccf3415858edef. 2023-09-06T20:35:21,791 DEBUG [RegionOpenAndInit-ns2:test-16940325181891-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16940325181891,,1694032520952.db7e5fe8ab24349786ccf3415858edef. after waiting 0 ms 2023-09-06T20:35:21,791 DEBUG [RegionOpenAndInit-ns2:test-16940325181891-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16940325181891,,1694032520952.db7e5fe8ab24349786ccf3415858edef. 2023-09-06T20:35:21,791 INFO [RegionOpenAndInit-ns2:test-16940325181891-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16940325181891,,1694032520952.db7e5fe8ab24349786ccf3415858edef. 2023-09-06T20:35:21,792 DEBUG [RegionOpenAndInit-ns2:test-16940325181891-pool-0 {}] regionserver.HRegion(1622): Region close journal for db7e5fe8ab24349786ccf3415858edef: 2023-09-06T20:35:21,793 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:test-16940325181891 execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:35:21,794 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16940325181891,,1694032520952.db7e5fe8ab24349786ccf3415858edef.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1694032521793"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032521793"}]},"ts":"1694032521793"} 2023-09-06T20:35:21,796 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:35:21,797 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:test-16940325181891 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:35:21,797 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16940325181891","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032521797"}]},"ts":"1694032521797"} 2023-09-06T20:35:21,799 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16940325181891, state=ENABLING in hbase:meta 2023-09-06T20:35:21,803 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16940325181891, region=db7e5fe8ab24349786ccf3415858edef, ASSIGN}] 2023-09-06T20:35:21,805 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16940325181891, region=db7e5fe8ab24349786ccf3415858edef, ASSIGN 2023-09-06T20:35:21,806 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16940325181891, region=db7e5fe8ab24349786ccf3415858edef, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36075,1694032504561; forceNewPlan=false, retain=false 2023-09-06T20:35:21,959 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=db7e5fe8ab24349786ccf3415858edef, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:21,961 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; OpenRegionProcedure db7e5fe8ab24349786ccf3415858edef, server=jenkins-hbase3.apache.org,36075,1694032504561}] 2023-09-06T20:35:22,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-06T20:35:22,118 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(130): Open ns2:test-16940325181891,,1694032520952.db7e5fe8ab24349786ccf3415858edef. 2023-09-06T20:35:22,118 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7254): Opening region: {ENCODED => db7e5fe8ab24349786ccf3415858edef, NAME => 'ns2:test-16940325181891,,1694032520952.db7e5fe8ab24349786ccf3415858edef.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:35:22,119 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:35:22,119 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16940325181891 db7e5fe8ab24349786ccf3415858edef 2023-09-06T20:35:22,119 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(888): Instantiated ns2:test-16940325181891,,1694032520952.db7e5fe8ab24349786ccf3415858edef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:22,119 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7296): checking encryption for db7e5fe8ab24349786ccf3415858edef 2023-09-06T20:35:22,119 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7299): checking classloading for db7e5fe8ab24349786ccf3415858edef 2023-09-06T20:35:22,121 INFO [StoreOpener-db7e5fe8ab24349786ccf3415858edef-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region db7e5fe8ab24349786ccf3415858edef 2023-09-06T20:35:22,123 INFO [StoreOpener-db7e5fe8ab24349786ccf3415858edef-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db7e5fe8ab24349786ccf3415858edef columnFamilyName f 2023-09-06T20:35:22,123 DEBUG [StoreOpener-db7e5fe8ab24349786ccf3415858edef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:22,124 INFO [StoreOpener-db7e5fe8ab24349786ccf3415858edef-1 {}] regionserver.HStore(324): Store=db7e5fe8ab24349786ccf3415858edef/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:35:22,125 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/ns2/test-16940325181891/db7e5fe8ab24349786ccf3415858edef 2023-09-06T20:35:22,126 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/ns2/test-16940325181891/db7e5fe8ab24349786ccf3415858edef 2023-09-06T20:35:22,129 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1076): writing seq id for db7e5fe8ab24349786ccf3415858edef 2023-09-06T20:35:22,140 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/ns2/test-16940325181891/db7e5fe8ab24349786ccf3415858edef/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:35:22,141 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): Opened db7e5fe8ab24349786ccf3415858edef; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10137256160, jitterRate=-0.05589444935321808}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:35:22,142 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(992): Region open journal for db7e5fe8ab24349786ccf3415858edef: 2023-09-06T20:35:22,144 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:test-16940325181891,,1694032520952.db7e5fe8ab24349786ccf3415858edef., pid=18, masterSystemTime=1694032522113 2023-09-06T20:35:22,146 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:test-16940325181891,,1694032520952.db7e5fe8ab24349786ccf3415858edef. 2023-09-06T20:35:22,147 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(158): Opened ns2:test-16940325181891,,1694032520952.db7e5fe8ab24349786ccf3415858edef. 2023-09-06T20:35:22,147 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=db7e5fe8ab24349786ccf3415858edef, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:22,154 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=18, resume processing ppid=17 2023-09-06T20:35:22,154 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=18, ppid=17, state=SUCCESS; OpenRegionProcedure db7e5fe8ab24349786ccf3415858edef, server=jenkins-hbase3.apache.org,36075,1694032504561 in 189 msec 2023-09-06T20:35:22,160 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=17, resume processing ppid=16 2023-09-06T20:35:22,160 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=17, ppid=16, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16940325181891, region=db7e5fe8ab24349786ccf3415858edef, ASSIGN in 352 msec 2023-09-06T20:35:22,161 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:test-16940325181891 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:35:22,161 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16940325181891","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032522161"}]},"ts":"1694032522161"} 2023-09-06T20:35:22,164 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16940325181891, state=ENABLED in hbase:meta 2023-09-06T20:35:22,168 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16940325181891 execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:35:22,170 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=16, state=SUCCESS; CreateTableProcedure table=ns2:test-16940325181891 in 1.2160 sec 2023-09-06T20:35:22,644 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16940325181891 2023-09-06T20:35:22,644 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-16940325181891 Metrics about Tables on a single HBase RegionServer 2023-09-06T20:35:22,646 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1694032518189 2023-09-06T20:35:22,646 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1694032518189 Metrics about Tables on a single HBase RegionServer 2023-09-06T20:35:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-06T20:35:23,072 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns2:test-16940325181891, procId: 16 completed 2023-09-06T20:35:23,079 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] regionserver.HRegion(8210): writing data to region ns2:test-16940325181891,,1694032520952.db7e5fe8ab24349786ccf3415858edef. with WAL disabled. Data may be lost in the event of a crash. 2023-09-06T20:35:23,174 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns3:test-16940325181892', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:35:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] procedure2.ProcedureExecutor(1032): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:test-16940325181892 2023-09-06T20:35:23,178 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16940325181892 execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:35:23,178 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:23,179 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "test-16940325181892" procId is: 19 2023-09-06T20:35:23,180 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:test-16940325181892 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:35:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-09-06T20:35:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-09-06T20:35:23,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-09-06T20:35:23,598 INFO [RegionOpenAndInit-ns3:test-16940325181892-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 10b75a50d5fcfa15e4a0da0ac3d85e01, NAME => 'ns3:test-16940325181892,,1694032523174.10b75a50d5fcfa15e4a0da0ac3d85e01.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16940325181892', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd 2023-09-06T20:35:23,622 DEBUG [RegionOpenAndInit-ns3:test-16940325181892-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16940325181892,,1694032523174.10b75a50d5fcfa15e4a0da0ac3d85e01.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:23,622 DEBUG [RegionOpenAndInit-ns3:test-16940325181892-pool-0 {}] regionserver.HRegion(1668): Closing 10b75a50d5fcfa15e4a0da0ac3d85e01, disabling compactions & flushes 2023-09-06T20:35:23,622 INFO [RegionOpenAndInit-ns3:test-16940325181892-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16940325181892,,1694032523174.10b75a50d5fcfa15e4a0da0ac3d85e01. 2023-09-06T20:35:23,622 DEBUG [RegionOpenAndInit-ns3:test-16940325181892-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16940325181892,,1694032523174.10b75a50d5fcfa15e4a0da0ac3d85e01. 2023-09-06T20:35:23,622 DEBUG [RegionOpenAndInit-ns3:test-16940325181892-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16940325181892,,1694032523174.10b75a50d5fcfa15e4a0da0ac3d85e01. after waiting 0 ms 2023-09-06T20:35:23,622 DEBUG [RegionOpenAndInit-ns3:test-16940325181892-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16940325181892,,1694032523174.10b75a50d5fcfa15e4a0da0ac3d85e01. 2023-09-06T20:35:23,623 INFO [RegionOpenAndInit-ns3:test-16940325181892-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16940325181892,,1694032523174.10b75a50d5fcfa15e4a0da0ac3d85e01. 2023-09-06T20:35:23,623 DEBUG [RegionOpenAndInit-ns3:test-16940325181892-pool-0 {}] regionserver.HRegion(1622): Region close journal for 10b75a50d5fcfa15e4a0da0ac3d85e01: 2023-09-06T20:35:23,624 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:test-16940325181892 execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:35:23,624 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16940325181892,,1694032523174.10b75a50d5fcfa15e4a0da0ac3d85e01.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1694032523624"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032523624"}]},"ts":"1694032523624"} 2023-09-06T20:35:23,627 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:35:23,628 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:test-16940325181892 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:35:23,628 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16940325181892","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032523628"}]},"ts":"1694032523628"} 2023-09-06T20:35:23,630 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16940325181892, state=ENABLING in hbase:meta 2023-09-06T20:35:23,634 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16940325181892, region=10b75a50d5fcfa15e4a0da0ac3d85e01, ASSIGN}] 2023-09-06T20:35:23,637 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16940325181892, region=10b75a50d5fcfa15e4a0da0ac3d85e01, ASSIGN 2023-09-06T20:35:23,638 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16940325181892, region=10b75a50d5fcfa15e4a0da0ac3d85e01, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36075,1694032504561; forceNewPlan=false, retain=false 2023-09-06T20:35:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-09-06T20:35:23,790 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=10b75a50d5fcfa15e4a0da0ac3d85e01, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:23,793 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; OpenRegionProcedure 10b75a50d5fcfa15e4a0da0ac3d85e01, server=jenkins-hbase3.apache.org,36075,1694032504561}] 2023-09-06T20:35:23,950 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(130): Open ns3:test-16940325181892,,1694032523174.10b75a50d5fcfa15e4a0da0ac3d85e01. 2023-09-06T20:35:23,950 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7254): Opening region: {ENCODED => 10b75a50d5fcfa15e4a0da0ac3d85e01, NAME => 'ns3:test-16940325181892,,1694032523174.10b75a50d5fcfa15e4a0da0ac3d85e01.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:35:23,951 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:35:23,951 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16940325181892 10b75a50d5fcfa15e4a0da0ac3d85e01 2023-09-06T20:35:23,951 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(888): Instantiated ns3:test-16940325181892,,1694032523174.10b75a50d5fcfa15e4a0da0ac3d85e01.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:23,951 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7296): checking encryption for 10b75a50d5fcfa15e4a0da0ac3d85e01 2023-09-06T20:35:23,951 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7299): checking classloading for 10b75a50d5fcfa15e4a0da0ac3d85e01 2023-09-06T20:35:23,953 INFO [StoreOpener-10b75a50d5fcfa15e4a0da0ac3d85e01-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 10b75a50d5fcfa15e4a0da0ac3d85e01 2023-09-06T20:35:23,955 INFO [StoreOpener-10b75a50d5fcfa15e4a0da0ac3d85e01-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 10b75a50d5fcfa15e4a0da0ac3d85e01 columnFamilyName f 2023-09-06T20:35:23,955 DEBUG [StoreOpener-10b75a50d5fcfa15e4a0da0ac3d85e01-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:23,955 INFO [StoreOpener-10b75a50d5fcfa15e4a0da0ac3d85e01-1 {}] regionserver.HStore(324): Store=10b75a50d5fcfa15e4a0da0ac3d85e01/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:35:23,957 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/ns3/test-16940325181892/10b75a50d5fcfa15e4a0da0ac3d85e01 2023-09-06T20:35:23,957 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/ns3/test-16940325181892/10b75a50d5fcfa15e4a0da0ac3d85e01 2023-09-06T20:35:23,960 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1076): writing seq id for 10b75a50d5fcfa15e4a0da0ac3d85e01 2023-09-06T20:35:23,963 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/ns3/test-16940325181892/10b75a50d5fcfa15e4a0da0ac3d85e01/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:35:23,964 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): Opened 10b75a50d5fcfa15e4a0da0ac3d85e01; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11438106400, jitterRate=0.06525667011737823}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:35:23,964 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(992): Region open journal for 10b75a50d5fcfa15e4a0da0ac3d85e01: 2023-09-06T20:35:23,966 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:test-16940325181892,,1694032523174.10b75a50d5fcfa15e4a0da0ac3d85e01., pid=21, masterSystemTime=1694032523945 2023-09-06T20:35:23,968 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:test-16940325181892,,1694032523174.10b75a50d5fcfa15e4a0da0ac3d85e01. 2023-09-06T20:35:23,968 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(158): Opened ns3:test-16940325181892,,1694032523174.10b75a50d5fcfa15e4a0da0ac3d85e01. 2023-09-06T20:35:23,969 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=10b75a50d5fcfa15e4a0da0ac3d85e01, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:23,974 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=21, resume processing ppid=20 2023-09-06T20:35:23,974 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=21, ppid=20, state=SUCCESS; OpenRegionProcedure 10b75a50d5fcfa15e4a0da0ac3d85e01, server=jenkins-hbase3.apache.org,36075,1694032504561 in 178 msec 2023-09-06T20:35:23,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=20, resume processing ppid=19 2023-09-06T20:35:23,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=20, ppid=19, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16940325181892, region=10b75a50d5fcfa15e4a0da0ac3d85e01, ASSIGN in 340 msec 2023-09-06T20:35:23,982 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:test-16940325181892 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:35:23,983 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16940325181892","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032523982"}]},"ts":"1694032523982"} 2023-09-06T20:35:23,984 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16940325181892, state=ENABLED in hbase:meta 2023-09-06T20:35:23,988 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16940325181892 execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:35:23,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=19, state=SUCCESS; CreateTableProcedure table=ns3:test-16940325181892 in 814 msec 2023-09-06T20:35:24,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-09-06T20:35:24,288 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns3:test-16940325181892, procId: 19 completed 2023-09-06T20:35:24,288 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns3:test-16940325181892 get assigned. Timeout = 60000ms 2023-09-06T20:35:24,290 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-06T20:35:24,294 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.HBaseTestingUtility(3581): All regions for table ns3:test-16940325181892 assigned to meta. Checking AM states. 2023-09-06T20:35:24,295 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-06T20:35:24,295 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.HBaseTestingUtility(3601): All regions for table ns3:test-16940325181892 assigned. 2023-09-06T20:35:24,297 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns4:test-16940325181893', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:35:24,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] procedure2.ProcedureExecutor(1032): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns4:test-16940325181893 2023-09-06T20:35:24,301 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16940325181893 execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:35:24,301 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:24,301 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns4" qualifier: "test-16940325181893" procId is: 22 2023-09-06T20:35:24,302 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns4:test-16940325181893 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:35:24,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-06T20:35:24,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-06T20:35:24,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-06T20:35:24,716 INFO [RegionOpenAndInit-ns4:test-16940325181893-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => f756fa51ee91914fe985b9228ffe9d4b, NAME => 'ns4:test-16940325181893,,1694032524297.f756fa51ee91914fe985b9228ffe9d4b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-16940325181893', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd 2023-09-06T20:35:24,874 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1694032518189' 2023-09-06T20:35:24,874 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-16940325181892' 2023-09-06T20:35:24,875 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-16940325181891' 2023-09-06T20:35:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-06T20:35:25,135 DEBUG [RegionOpenAndInit-ns4:test-16940325181893-pool-0 {}] regionserver.HRegion(888): Instantiated ns4:test-16940325181893,,1694032524297.f756fa51ee91914fe985b9228ffe9d4b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:25,135 DEBUG [RegionOpenAndInit-ns4:test-16940325181893-pool-0 {}] regionserver.HRegion(1668): Closing f756fa51ee91914fe985b9228ffe9d4b, disabling compactions & flushes 2023-09-06T20:35:25,135 INFO [RegionOpenAndInit-ns4:test-16940325181893-pool-0 {}] regionserver.HRegion(1690): Closing region ns4:test-16940325181893,,1694032524297.f756fa51ee91914fe985b9228ffe9d4b. 2023-09-06T20:35:25,135 DEBUG [RegionOpenAndInit-ns4:test-16940325181893-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16940325181893,,1694032524297.f756fa51ee91914fe985b9228ffe9d4b. 2023-09-06T20:35:25,135 DEBUG [RegionOpenAndInit-ns4:test-16940325181893-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16940325181893,,1694032524297.f756fa51ee91914fe985b9228ffe9d4b. after waiting 0 ms 2023-09-06T20:35:25,135 DEBUG [RegionOpenAndInit-ns4:test-16940325181893-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16940325181893,,1694032524297.f756fa51ee91914fe985b9228ffe9d4b. 2023-09-06T20:35:25,135 INFO [RegionOpenAndInit-ns4:test-16940325181893-pool-0 {}] regionserver.HRegion(1909): Closed ns4:test-16940325181893,,1694032524297.f756fa51ee91914fe985b9228ffe9d4b. 2023-09-06T20:35:25,135 DEBUG [RegionOpenAndInit-ns4:test-16940325181893-pool-0 {}] regionserver.HRegion(1622): Region close journal for f756fa51ee91914fe985b9228ffe9d4b: 2023-09-06T20:35:25,137 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns4:test-16940325181893 execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:35:25,137 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns4:test-16940325181893,,1694032524297.f756fa51ee91914fe985b9228ffe9d4b.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1694032525137"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032525137"}]},"ts":"1694032525137"} 2023-09-06T20:35:25,139 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:35:25,140 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns4:test-16940325181893 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:35:25,140 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16940325181893","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032525140"}]},"ts":"1694032525140"} 2023-09-06T20:35:25,142 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16940325181893, state=ENABLING in hbase:meta 2023-09-06T20:35:25,145 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16940325181893, region=f756fa51ee91914fe985b9228ffe9d4b, ASSIGN}] 2023-09-06T20:35:25,148 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16940325181893, region=f756fa51ee91914fe985b9228ffe9d4b, ASSIGN 2023-09-06T20:35:25,149 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns4:test-16940325181893, region=f756fa51ee91914fe985b9228ffe9d4b, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36075,1694032504561; forceNewPlan=false, retain=false 2023-09-06T20:35:25,301 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=f756fa51ee91914fe985b9228ffe9d4b, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:25,304 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE; OpenRegionProcedure f756fa51ee91914fe985b9228ffe9d4b, server=jenkins-hbase3.apache.org,36075,1694032504561}] 2023-09-06T20:35:25,383 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:35:25,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-06T20:35:25,462 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(130): Open ns4:test-16940325181893,,1694032524297.f756fa51ee91914fe985b9228ffe9d4b. 2023-09-06T20:35:25,462 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7254): Opening region: {ENCODED => f756fa51ee91914fe985b9228ffe9d4b, NAME => 'ns4:test-16940325181893,,1694032524297.f756fa51ee91914fe985b9228ffe9d4b.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:35:25,462 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:35:25,462 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16940325181893 f756fa51ee91914fe985b9228ffe9d4b 2023-09-06T20:35:25,463 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(888): Instantiated ns4:test-16940325181893,,1694032524297.f756fa51ee91914fe985b9228ffe9d4b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:25,463 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7296): checking encryption for f756fa51ee91914fe985b9228ffe9d4b 2023-09-06T20:35:25,463 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7299): checking classloading for f756fa51ee91914fe985b9228ffe9d4b 2023-09-06T20:35:25,464 INFO [StoreOpener-f756fa51ee91914fe985b9228ffe9d4b-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region f756fa51ee91914fe985b9228ffe9d4b 2023-09-06T20:35:25,467 INFO [StoreOpener-f756fa51ee91914fe985b9228ffe9d4b-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f756fa51ee91914fe985b9228ffe9d4b columnFamilyName f 2023-09-06T20:35:25,467 DEBUG [StoreOpener-f756fa51ee91914fe985b9228ffe9d4b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:25,468 INFO [StoreOpener-f756fa51ee91914fe985b9228ffe9d4b-1 {}] regionserver.HStore(324): Store=f756fa51ee91914fe985b9228ffe9d4b/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:35:25,469 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/ns4/test-16940325181893/f756fa51ee91914fe985b9228ffe9d4b 2023-09-06T20:35:25,469 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/ns4/test-16940325181893/f756fa51ee91914fe985b9228ffe9d4b 2023-09-06T20:35:25,477 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1076): writing seq id for f756fa51ee91914fe985b9228ffe9d4b 2023-09-06T20:35:25,489 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/ns4/test-16940325181893/f756fa51ee91914fe985b9228ffe9d4b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:35:25,490 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): Opened f756fa51ee91914fe985b9228ffe9d4b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10648981440, jitterRate=-0.008236318826675415}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:35:25,491 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(992): Region open journal for f756fa51ee91914fe985b9228ffe9d4b: 2023-09-06T20:35:25,493 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2556): Post open deploy tasks for ns4:test-16940325181893,,1694032524297.f756fa51ee91914fe985b9228ffe9d4b., pid=24, masterSystemTime=1694032525457 2023-09-06T20:35:25,495 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2583): Finished post open deploy task for ns4:test-16940325181893,,1694032524297.f756fa51ee91914fe985b9228ffe9d4b. 2023-09-06T20:35:25,495 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(158): Opened ns4:test-16940325181893,,1694032524297.f756fa51ee91914fe985b9228ffe9d4b. 2023-09-06T20:35:25,496 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=f756fa51ee91914fe985b9228ffe9d4b, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:25,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=24, resume processing ppid=23 2023-09-06T20:35:25,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=24, ppid=23, state=SUCCESS; OpenRegionProcedure f756fa51ee91914fe985b9228ffe9d4b, server=jenkins-hbase3.apache.org,36075,1694032504561 in 194 msec 2023-09-06T20:35:25,503 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=23, resume processing ppid=22 2023-09-06T20:35:25,504 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=23, ppid=22, state=SUCCESS; TransitRegionStateProcedure table=ns4:test-16940325181893, region=f756fa51ee91914fe985b9228ffe9d4b, ASSIGN in 356 msec 2023-09-06T20:35:25,504 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns4:test-16940325181893 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:35:25,505 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16940325181893","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032525504"}]},"ts":"1694032525504"} 2023-09-06T20:35:25,506 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16940325181893, state=ENABLED in hbase:meta 2023-09-06T20:35:25,509 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16940325181893 execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:35:25,510 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=22, state=SUCCESS; CreateTableProcedure table=ns4:test-16940325181893 in 1.2120 sec 2023-09-06T20:35:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-06T20:35:26,412 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns4:test-16940325181893, procId: 22 completed 2023-09-06T20:35:26,412 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns4:test-16940325181893 get assigned. Timeout = 60000ms 2023-09-06T20:35:26,412 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-06T20:35:26,415 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.HBaseTestingUtility(3581): All regions for table ns4:test-16940325181893 assigned to meta. Checking AM states. 2023-09-06T20:35:26,415 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-06T20:35:26,416 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.HBaseTestingUtility(3601): All regions for table ns4:test-16940325181893 assigned. 2023-09-06T20:35:26,416 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1191cd9f to 127.0.0.1:50423 2023-09-06T20:35:26,416 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:26,480 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.ResourceChecker(147): before: backup.TestRemoteRestore#testFullRestoreRemote Thread=815, OpenFileDescriptor=881, MaxFileDescriptor=60000, SystemLoadAverage=161, ProcessCount=167, AvailableMemoryMB=4025 2023-09-06T20:35:26,480 WARN [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.ResourceChecker(130): Thread=815 is superior to 500 2023-09-06T20:35:26,482 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] backup.TestRemoteRestore(70): test remote full backup on a single table 2023-09-06T20:35:26,483 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0ba73ac6 to 127.0.0.1:50423 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:35:26,489 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1298b2b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:26,511 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:26,513 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:60812, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:26,527 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:35:26,529 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:44886, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:35:26,542 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'backup'} 2023-09-06T20:35:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] procedure2.ProcedureExecutor(1032): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=backup 2023-09-06T20:35:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=25 2023-09-06T20:35:26,554 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:35:26,563 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=25, state=SUCCESS; CreateNamespaceProcedure, namespace=backup in 19 msec 2023-09-06T20:35:26,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=25 2023-09-06T20:35:26,663 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:35:26,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] procedure2.ProcedureExecutor(1032): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system 2023-09-06T20:35:26,668 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:35:26,669 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:26,669 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2023-09-06T20:35:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-09-06T20:35:26,670 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:35:26,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-09-06T20:35:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-09-06T20:35:27,083 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 8e818fe31858c9c4a15e134b8245804a, NAME => 'backup:system,,1694032526663.8e818fe31858c9c4a15e134b8245804a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd 2023-09-06T20:35:27,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-09-06T20:35:27,495 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system,,1694032526663.8e818fe31858c9c4a15e134b8245804a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:27,495 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1668): Closing 8e818fe31858c9c4a15e134b8245804a, disabling compactions & flushes 2023-09-06T20:35:27,495 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system,,1694032526663.8e818fe31858c9c4a15e134b8245804a. 2023-09-06T20:35:27,495 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1694032526663.8e818fe31858c9c4a15e134b8245804a. 2023-09-06T20:35:27,495 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1694032526663.8e818fe31858c9c4a15e134b8245804a. after waiting 0 ms 2023-09-06T20:35:27,495 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1694032526663.8e818fe31858c9c4a15e134b8245804a. 2023-09-06T20:35:27,495 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1909): Closed backup:system,,1694032526663.8e818fe31858c9c4a15e134b8245804a. 2023-09-06T20:35:27,495 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1622): Region close journal for 8e818fe31858c9c4a15e134b8245804a: 2023-09-06T20:35:27,497 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:35:27,497 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1694032526663.8e818fe31858c9c4a15e134b8245804a.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1694032527497"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032527497"}]},"ts":"1694032527497"} 2023-09-06T20:35:27,499 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:35:27,501 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:35:27,501 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032527501"}]},"ts":"1694032527501"} 2023-09-06T20:35:27,502 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-09-06T20:35:27,505 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=8e818fe31858c9c4a15e134b8245804a, ASSIGN}] 2023-09-06T20:35:27,508 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=8e818fe31858c9c4a15e134b8245804a, ASSIGN 2023-09-06T20:35:27,509 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=8e818fe31858c9c4a15e134b8245804a, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36075,1694032504561; forceNewPlan=false, retain=false 2023-09-06T20:35:27,661 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=8e818fe31858c9c4a15e134b8245804a, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:27,663 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; OpenRegionProcedure 8e818fe31858c9c4a15e134b8245804a, server=jenkins-hbase3.apache.org,36075,1694032504561}] 2023-09-06T20:35:27,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-09-06T20:35:27,820 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(130): Open backup:system,,1694032526663.8e818fe31858c9c4a15e134b8245804a. 2023-09-06T20:35:27,820 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7254): Opening region: {ENCODED => 8e818fe31858c9c4a15e134b8245804a, NAME => 'backup:system,,1694032526663.8e818fe31858c9c4a15e134b8245804a.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:35:27,821 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:35:27,821 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 8e818fe31858c9c4a15e134b8245804a 2023-09-06T20:35:27,821 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(888): Instantiated backup:system,,1694032526663.8e818fe31858c9c4a15e134b8245804a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:27,821 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7296): checking encryption for 8e818fe31858c9c4a15e134b8245804a 2023-09-06T20:35:27,821 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7299): checking classloading for 8e818fe31858c9c4a15e134b8245804a 2023-09-06T20:35:27,823 INFO [StoreOpener-8e818fe31858c9c4a15e134b8245804a-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 8e818fe31858c9c4a15e134b8245804a 2023-09-06T20:35:27,825 INFO [StoreOpener-8e818fe31858c9c4a15e134b8245804a-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e818fe31858c9c4a15e134b8245804a columnFamilyName meta 2023-09-06T20:35:27,825 DEBUG [StoreOpener-8e818fe31858c9c4a15e134b8245804a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:27,825 INFO [StoreOpener-8e818fe31858c9c4a15e134b8245804a-1 {}] regionserver.HStore(324): Store=8e818fe31858c9c4a15e134b8245804a/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:35:27,826 INFO [StoreOpener-8e818fe31858c9c4a15e134b8245804a-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 8e818fe31858c9c4a15e134b8245804a 2023-09-06T20:35:27,827 INFO [StoreOpener-8e818fe31858c9c4a15e134b8245804a-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e818fe31858c9c4a15e134b8245804a columnFamilyName session 2023-09-06T20:35:27,828 DEBUG [StoreOpener-8e818fe31858c9c4a15e134b8245804a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:27,830 INFO [StoreOpener-8e818fe31858c9c4a15e134b8245804a-1 {}] regionserver.HStore(324): Store=8e818fe31858c9c4a15e134b8245804a/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:35:27,831 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a 2023-09-06T20:35:27,831 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a 2023-09-06T20:35:27,833 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-09-06T20:35:27,834 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1076): writing seq id for 8e818fe31858c9c4a15e134b8245804a 2023-09-06T20:35:27,837 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:35:27,837 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): Opened 8e818fe31858c9c4a15e134b8245804a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11003004480, jitterRate=0.02473464608192444}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-09-06T20:35:27,838 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(992): Region open journal for 8e818fe31858c9c4a15e134b8245804a: 2023-09-06T20:35:27,839 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1694032526663.8e818fe31858c9c4a15e134b8245804a., pid=28, masterSystemTime=1694032527815 2023-09-06T20:35:27,842 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1694032526663.8e818fe31858c9c4a15e134b8245804a. 2023-09-06T20:35:27,842 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(158): Opened backup:system,,1694032526663.8e818fe31858c9c4a15e134b8245804a. 2023-09-06T20:35:27,842 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=8e818fe31858c9c4a15e134b8245804a, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:27,846 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=28, resume processing ppid=27 2023-09-06T20:35:27,846 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=28, ppid=27, state=SUCCESS; OpenRegionProcedure 8e818fe31858c9c4a15e134b8245804a, server=jenkins-hbase3.apache.org,36075,1694032504561 in 181 msec 2023-09-06T20:35:27,849 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=27, resume processing ppid=26 2023-09-06T20:35:27,849 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=27, ppid=26, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=8e818fe31858c9c4a15e134b8245804a, ASSIGN in 341 msec 2023-09-06T20:35:27,850 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:35:27,850 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032527850"}]},"ts":"1694032527850"} 2023-09-06T20:35:27,852 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-09-06T20:35:27,854 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:35:27,856 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=26, state=SUCCESS; CreateTableProcedure table=backup:system in 1.1910 sec 2023-09-06T20:35:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-09-06T20:35:28,777 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: backup:system, procId: 26 completed 2023-09-06T20:35:28,791 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:35:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] procedure2.ProcedureExecutor(1032): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system_bulk 2023-09-06T20:35:28,795 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:35:28,796 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:28,796 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2023-09-06T20:35:28,797 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:35:28,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-09-06T20:35:28,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-09-06T20:35:29,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-09-06T20:35:29,210 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 57404e38a649f5014400023c3ebc14c3, NAME => 'backup:system_bulk,,1694032528791.57404e38a649f5014400023c3ebc14c3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd 2023-09-06T20:35:29,221 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1694032528791.57404e38a649f5014400023c3ebc14c3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:29,221 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1668): Closing 57404e38a649f5014400023c3ebc14c3, disabling compactions & flushes 2023-09-06T20:35:29,221 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1694032528791.57404e38a649f5014400023c3ebc14c3. 2023-09-06T20:35:29,221 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1694032528791.57404e38a649f5014400023c3ebc14c3. 2023-09-06T20:35:29,221 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1694032528791.57404e38a649f5014400023c3ebc14c3. after waiting 0 ms 2023-09-06T20:35:29,222 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1694032528791.57404e38a649f5014400023c3ebc14c3. 2023-09-06T20:35:29,222 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1909): Closed backup:system_bulk,,1694032528791.57404e38a649f5014400023c3ebc14c3. 2023-09-06T20:35:29,222 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1622): Region close journal for 57404e38a649f5014400023c3ebc14c3: 2023-09-06T20:35:29,223 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:35:29,223 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system_bulk,,1694032528791.57404e38a649f5014400023c3ebc14c3.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1694032529223"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032529223"}]},"ts":"1694032529223"} 2023-09-06T20:35:29,225 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:35:29,226 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:35:29,227 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032529227"}]},"ts":"1694032529227"} 2023-09-06T20:35:29,228 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2023-09-06T20:35:29,232 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=57404e38a649f5014400023c3ebc14c3, ASSIGN}] 2023-09-06T20:35:29,235 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=57404e38a649f5014400023c3ebc14c3, ASSIGN 2023-09-06T20:35:29,236 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system_bulk, region=57404e38a649f5014400023c3ebc14c3, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36075,1694032504561; forceNewPlan=false, retain=false 2023-09-06T20:35:29,388 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=57404e38a649f5014400023c3ebc14c3, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:29,390 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE; OpenRegionProcedure 57404e38a649f5014400023c3ebc14c3, server=jenkins-hbase3.apache.org,36075,1694032504561}] 2023-09-06T20:35:29,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-09-06T20:35:29,547 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(130): Open backup:system_bulk,,1694032528791.57404e38a649f5014400023c3ebc14c3. 2023-09-06T20:35:29,547 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7254): Opening region: {ENCODED => 57404e38a649f5014400023c3ebc14c3, NAME => 'backup:system_bulk,,1694032528791.57404e38a649f5014400023c3ebc14c3.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:35:29,547 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:35:29,547 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk 57404e38a649f5014400023c3ebc14c3 2023-09-06T20:35:29,548 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1694032528791.57404e38a649f5014400023c3ebc14c3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:29,548 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7296): checking encryption for 57404e38a649f5014400023c3ebc14c3 2023-09-06T20:35:29,548 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7299): checking classloading for 57404e38a649f5014400023c3ebc14c3 2023-09-06T20:35:29,549 INFO [StoreOpener-57404e38a649f5014400023c3ebc14c3-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 57404e38a649f5014400023c3ebc14c3 2023-09-06T20:35:29,551 INFO [StoreOpener-57404e38a649f5014400023c3ebc14c3-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 57404e38a649f5014400023c3ebc14c3 columnFamilyName meta 2023-09-06T20:35:29,551 DEBUG [StoreOpener-57404e38a649f5014400023c3ebc14c3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:29,551 INFO [StoreOpener-57404e38a649f5014400023c3ebc14c3-1 {}] regionserver.HStore(324): Store=57404e38a649f5014400023c3ebc14c3/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:35:29,551 INFO [StoreOpener-57404e38a649f5014400023c3ebc14c3-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 57404e38a649f5014400023c3ebc14c3 2023-09-06T20:35:29,553 INFO [StoreOpener-57404e38a649f5014400023c3ebc14c3-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 57404e38a649f5014400023c3ebc14c3 columnFamilyName session 2023-09-06T20:35:29,553 DEBUG [StoreOpener-57404e38a649f5014400023c3ebc14c3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:29,554 INFO [StoreOpener-57404e38a649f5014400023c3ebc14c3-1 {}] regionserver.HStore(324): Store=57404e38a649f5014400023c3ebc14c3/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:35:29,554 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system_bulk/57404e38a649f5014400023c3ebc14c3 2023-09-06T20:35:29,555 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system_bulk/57404e38a649f5014400023c3ebc14c3 2023-09-06T20:35:29,556 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-09-06T20:35:29,558 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1076): writing seq id for 57404e38a649f5014400023c3ebc14c3 2023-09-06T20:35:29,561 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system_bulk/57404e38a649f5014400023c3ebc14c3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:35:29,562 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): Opened 57404e38a649f5014400023c3ebc14c3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10948191200, jitterRate=0.019629761576652527}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-09-06T20:35:29,563 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(992): Region open journal for 57404e38a649f5014400023c3ebc14c3: 2023-09-06T20:35:29,564 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system_bulk,,1694032528791.57404e38a649f5014400023c3ebc14c3., pid=31, masterSystemTime=1694032529543 2023-09-06T20:35:29,566 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system_bulk,,1694032528791.57404e38a649f5014400023c3ebc14c3. 2023-09-06T20:35:29,566 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(158): Opened backup:system_bulk,,1694032528791.57404e38a649f5014400023c3ebc14c3. 2023-09-06T20:35:29,566 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=57404e38a649f5014400023c3ebc14c3, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:29,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=31, resume processing ppid=30 2023-09-06T20:35:29,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=31, ppid=30, state=SUCCESS; OpenRegionProcedure 57404e38a649f5014400023c3ebc14c3, server=jenkins-hbase3.apache.org,36075,1694032504561 in 178 msec 2023-09-06T20:35:29,573 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=30, resume processing ppid=29 2023-09-06T20:35:29,573 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=30, ppid=29, state=SUCCESS; TransitRegionStateProcedure table=backup:system_bulk, region=57404e38a649f5014400023c3ebc14c3, ASSIGN in 339 msec 2023-09-06T20:35:29,574 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:35:29,575 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032529574"}]},"ts":"1694032529574"} 2023-09-06T20:35:29,576 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2023-09-06T20:35:29,580 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:35:29,582 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=29, state=SUCCESS; CreateTableProcedure table=backup:system_bulk in 789 msec 2023-09-06T20:35:29,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-09-06T20:35:29,903 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: backup:system_bulk, procId: 29 completed 2023-09-06T20:35:29,918 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-09-06T20:35:29,972 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:35:29,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032529972 (current time:1694032529972). 2023-09-06T20:35:29,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:35:29,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-09-06T20:35:29,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7650ea14 to 127.0.0.1:50423 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:35:29,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@591909a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:30,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:30,004 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:60826, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:30,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7650ea14 to 127.0.0.1:50423 2023-09-06T20:35:30,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:30,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:35:30,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] procedure2.ProcedureExecutor(1032): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:35:30,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-09-06T20:35:30,030 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:35:30,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-09-06T20:35:30,037 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:35:30,041 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:35:30,059 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:35:30,062 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 8e818fe31858c9c4a15e134b8245804a}] 2023-09-06T20:35:30,070 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 8e818fe31858c9c4a15e134b8245804a 2023-09-06T20:35:30,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-09-06T20:35:30,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36075 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2023-09-06T20:35:30,234 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1694032526663.8e818fe31858c9c4a15e134b8245804a. 2023-09-06T20:35:30,237 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2819): Flushing 8e818fe31858c9c4a15e134b8245804a 2/2 column families, dataSize=45 B heapSize=632 B 2023-09-06T20:35:30,317 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/.tmp/session/db85f23ebe2a4f8b9aaa70d53427ac0d is 49, key is activesession:/session:c/1694032529956/Put/seqid=0 2023-09-06T20:35:30,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-09-06T20:35:30,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-09-06T20:35:30,741 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/.tmp/session/db85f23ebe2a4f8b9aaa70d53427ac0d 2023-09-06T20:35:30,812 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/.tmp/session/db85f23ebe2a4f8b9aaa70d53427ac0d as hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/session/db85f23ebe2a4f8b9aaa70d53427ac0d 2023-09-06T20:35:30,820 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/session/db85f23ebe2a4f8b9aaa70d53427ac0d, entries=1, sequenceid=5, filesize=4.9 K 2023-09-06T20:35:30,824 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3022): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for 8e818fe31858c9c4a15e134b8245804a in 587ms, sequenceid=5, compaction requested=false 2023-09-06T20:35:30,824 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-09-06T20:35:30,826 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2520): Flush status journal for 8e818fe31858c9c4a15e134b8245804a: 2023-09-06T20:35:30,826 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1694032526663.8e818fe31858c9c4a15e134b8245804a. for snapshot_backup_system completed. 2023-09-06T20:35:30,827 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1694032526663.8e818fe31858c9c4a15e134b8245804a.' region-info for snapshot=snapshot_backup_system 2023-09-06T20:35:30,834 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:35:30,839 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-09-06T20:35:30,839 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/session/db85f23ebe2a4f8b9aaa70d53427ac0d] hfiles 2023-09-06T20:35:30,839 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/session/db85f23ebe2a4f8b9aaa70d53427ac0d for snapshot=snapshot_backup_system 2023-09-06T20:35:30,965 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:35:31,083 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2023-09-06T20:35:31,084 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-16940325181893' 2023-09-06T20:35:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-09-06T20:35:31,255 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1694032526663.8e818fe31858c9c4a15e134b8245804a. 2023-09-06T20:35:31,259 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=33 2023-09-06T20:35:31,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.HMaster(3995): Remote procedure done, pid=33 2023-09-06T20:35:31,263 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 8e818fe31858c9c4a15e134b8245804a 2023-09-06T20:35:31,265 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 8e818fe31858c9c4a15e134b8245804a 2023-09-06T20:35:31,269 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=33, resume processing ppid=32 2023-09-06T20:35:31,269 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=33, ppid=32, state=SUCCESS; SnapshotRegionProcedure 8e818fe31858c9c4a15e134b8245804a in 1.2050 sec 2023-09-06T20:35:31,269 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:35:31,271 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:35:31,275 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:35:31,275 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-09-06T20:35:31,278 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/.hbase-snapshot/.tmp/snapshot_backup_system 2023-09-06T20:35:31,705 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:35:31,717 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:35:31,718 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/.hbase-snapshot/snapshot_backup_system 2023-09-06T20:35:31,720 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:35:31,720 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-09-06T20:35:31,721 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=32, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.7020 sec 2023-09-06T20:35:32,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-09-06T20:35:32,145 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: backup:system, procId: 32 completed 2023-09-06T20:35:32,146 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] impl.TableBackupClient(115): Backup backup_1694032526492 started at 1694032532145. 2023-09-06T20:35:32,177 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.TableBackupClient(119): Backup session backup_1694032526492 has been started. 2023-09-06T20:35:32,180 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-09-06T20:35:32,194 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-09-06T20:35:32,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-09-06T20:35:32,203 INFO [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-09-06T20:35:32,203 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-09-06T20:35:32,204 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-09-06T20:35:32,204 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-09-06T20:35:32,205 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-09-06T20:35:32,205 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:35:32,206 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-09-06T20:35:32,206 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:32,206 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:35:32,206 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:35:32,207 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:32,207 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-09-06T20:35:32,207 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-09-06T20:35:32,207 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-09-06T20:35:32,208 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-09-06T20:35:32,208 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:35:32,208 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-09-06T20:35:32,211 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-09-06T20:35:32,211 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-09-06T20:35:32,212 DEBUG [member: 'jenkins-hbase3.apache.org,36075,1694032504561' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-09-06T20:35:32,212 DEBUG [member: 'jenkins-hbase3.apache.org,36075,1694032504561' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-09-06T20:35:32,213 DEBUG [member: 'jenkins-hbase3.apache.org,36075,1694032504561' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-09-06T20:35:32,213 DEBUG [member: 'jenkins-hbase3.apache.org,36075,1694032504561' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-09-06T20:35:32,213 DEBUG [member: 'jenkins-hbase3.apache.org,36075,1694032504561' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,36075,1694032504561' joining acquired barrier for procedure (rolllog) in zk 2023-09-06T20:35:32,216 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:32,216 DEBUG [member: 'jenkins-hbase3.apache.org,36075,1694032504561' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-09-06T20:35:32,216 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:32,216 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:35:32,216 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:35:32,216 DEBUG [member: 'jenkins-hbase3.apache.org,36075,1694032504561' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-09-06T20:35:32,217 DEBUG [member: 'jenkins-hbase3.apache.org,36075,1694032504561' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-09-06T20:35:32,217 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:35:32,217 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:35:32,217 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:35:32,218 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:32,218 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:35:32,218 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,36075,1694032504561' joining acquired barrier for procedure 'rolllog' on coordinator 2023-09-06T20:35:32,219 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@c80d444[Count = 0] remaining members to acquire global barrier 2023-09-06T20:35:32,219 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-09-06T20:35:32,219 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-09-06T20:35:32,220 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-09-06T20:35:32,220 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-09-06T20:35:32,220 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-09-06T20:35:32,220 DEBUG [member: 'jenkins-hbase3.apache.org,36075,1694032504561' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-09-06T20:35:32,220 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:32,220 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-09-06T20:35:32,222 DEBUG [member: 'jenkins-hbase3.apache.org,36075,1694032504561' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-09-06T20:35:32,222 DEBUG [rs(jenkins-hbase3.apache.org,36075,1694032504561)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:32,222 INFO [rs(jenkins-hbase3.apache.org,36075,1694032504561)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1694032506709 highest: 1694032506709 on jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:32,222 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36075%2C1694032504561:(num 1694032506709) roll requested 2023-09-06T20:35:32,239 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/WALs/jenkins-hbase3.apache.org,36075,1694032504561/jenkins-hbase3.apache.org%2C36075%2C1694032504561.1694032532222, exclude list is [], retry=0 2023-09-06T20:35:32,248 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46843,DS-1307a321-484a-4087-acdc-931715071dc3,DISK] 2023-09-06T20:35:32,255 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/WALs/jenkins-hbase3.apache.org,36075,1694032504561/jenkins-hbase3.apache.org%2C36075%2C1694032504561.1694032506709 with entries=19, filesize=3.99 KB; new WAL /user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/WALs/jenkins-hbase3.apache.org,36075,1694032504561/jenkins-hbase3.apache.org%2C36075%2C1694032504561.1694032532222 2023-09-06T20:35:32,256 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:46843,DS-1307a321-484a-4087-acdc-931715071dc3,DISK]] 2023-09-06T20:35:32,257 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/WALs/jenkins-hbase3.apache.org,36075,1694032504561/jenkins-hbase3.apache.org%2C36075%2C1694032504561.1694032506709 is not closed yet, will try archiving it next time 2023-09-06T20:35:32,257 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36075%2C1694032504561.meta:.meta(num 1694032507208) roll requested 2023-09-06T20:35:32,271 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/WALs/jenkins-hbase3.apache.org,36075,1694032504561/jenkins-hbase3.apache.org%2C36075%2C1694032504561.meta.1694032532257.meta, exclude list is [], retry=0 2023-09-06T20:35:32,275 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46843,DS-1307a321-484a-4087-acdc-931715071dc3,DISK] 2023-09-06T20:35:32,278 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/WALs/jenkins-hbase3.apache.org,36075,1694032504561/jenkins-hbase3.apache.org%2C36075%2C1694032504561.meta.1694032507208.meta with entries=36, filesize=11.13 KB; new WAL /user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/WALs/jenkins-hbase3.apache.org,36075,1694032504561/jenkins-hbase3.apache.org%2C36075%2C1694032504561.meta.1694032532257.meta 2023-09-06T20:35:32,278 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:46843,DS-1307a321-484a-4087-acdc-931715071dc3,DISK]] 2023-09-06T20:35:32,278 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/WALs/jenkins-hbase3.apache.org,36075,1694032504561/jenkins-hbase3.apache.org%2C36075%2C1694032504561.meta.1694032507208.meta is not closed yet, will try archiving it next time 2023-09-06T20:35:32,283 DEBUG [rs(jenkins-hbase3.apache.org,36075,1694032504561)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2023-09-06T20:35:32,283 INFO [rs(jenkins-hbase3.apache.org,36075,1694032504561)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1694032532222 on jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:32,283 DEBUG [rs(jenkins-hbase3.apache.org,36075,1694032504561)-backup-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:35:32,285 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51596, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-09-06T20:35:32,302 DEBUG [member: 'jenkins-hbase3.apache.org,36075,1694032504561' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-09-06T20:35:32,302 DEBUG [member: 'jenkins-hbase3.apache.org,36075,1694032504561' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,36075,1694032504561' in zk 2023-09-06T20:35:32,304 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:32,304 DEBUG [member: 'jenkins-hbase3.apache.org,36075,1694032504561' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-09-06T20:35:32,304 DEBUG [member: 'jenkins-hbase3.apache.org,36075,1694032504561' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:35:32,304 DEBUG [member: 'jenkins-hbase3.apache.org,36075,1694032504561' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-09-06T20:35:32,304 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:32,305 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:35:32,305 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:35:32,305 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:35:32,306 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:35:32,306 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:35:32,306 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:32,306 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:35:32,307 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:35:32,307 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:32,308 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,36075,1694032504561': 2023-09-06T20:35:32,308 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,36075,1694032504561' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-09-06T20:35:32,308 INFO [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-09-06T20:35:32,308 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-09-06T20:35:32,308 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-09-06T20:35:32,308 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-09-06T20:35:32,308 INFO [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-09-06T20:35:32,310 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-09-06T20:35:32,310 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-09-06T20:35:32,310 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-09-06T20:35:32,310 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-09-06T20:35:32,310 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-09-06T20:35:32,310 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:35:32,310 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:35:32,310 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:35:32,310 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:35:32,310 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:35:32,310 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:35:32,310 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:32,311 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:35:32,311 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:35:32,311 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:35:32,311 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:35:32,312 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:32,312 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:32,312 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:35:32,312 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:35:32,313 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:32,319 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:32,319 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-09-06T20:35:32,319 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-09-06T20:35:32,320 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-09-06T20:35:32,320 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-09-06T20:35:32,320 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:35:32,320 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:32,320 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:35:32,320 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-09-06T20:35:32,320 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:35:32,320 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-09-06T20:35:32,320 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-09-06T20:35:32,321 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-09-06T20:35:32,321 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-09-06T20:35:32,321 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:35:32,321 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:35:32,323 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] client.HBaseAdmin(2838): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-09-06T20:35:32,323 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] client.HBaseAdmin(2846): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-09-06T20:35:32,423 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] client.HBaseAdmin(2852): Getting current status of procedure from master... 2023-09-06T20:35:32,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-09-06T20:35:32,434 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1694032532433_default_test-1694032518189 table=test-1694032518189 type=FLUSH ttl=0 } 2023-09-06T20:35:32,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032532434 (current time:1694032532434). 2023-09-06T20:35:32,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:35:32,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1694032532433_default_test-1694032518189 VERSION not specified, setting to 2 2023-09-06T20:35:32,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x59ab8a02 to 127.0.0.1:50423 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:35:32,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e9d40e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:32,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:32,445 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49006, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:32,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x59ab8a02 to 127.0.0.1:50423 2023-09-06T20:35:32,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:32,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:35:32,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] procedure2.ProcedureExecutor(1032): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032532433_default_test-1694032518189 table=test-1694032518189 type=FLUSH ttl=0 } 2023-09-06T20:35:32,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1694032532433_default_test-1694032518189 table=test-1694032518189 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-09-06T20:35:32,450 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032532433_default_test-1694032518189 table=test-1694032518189 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:35:32,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-06T20:35:32,451 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032532433_default_test-1694032518189 table=test-1694032518189 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:35:32,453 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032532433_default_test-1694032518189 table=test-1694032518189 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:35:32,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-06T20:35:32,644 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-09-06T20:35:32,644 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-09-06T20:35:32,645 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16940325181893 2023-09-06T20:35:32,645 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-16940325181893 Metrics about Tables on a single HBase RegionServer 2023-09-06T20:35:32,646 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-09-06T20:35:32,646 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2023-09-06T20:35:32,647 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16940325181892 2023-09-06T20:35:32,647 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-16940325181892 Metrics about Tables on a single HBase RegionServer 2023-09-06T20:35:32,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-06T20:35:32,866 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032532433_default_test-1694032518189 table=test-1694032518189 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:35:32,866 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure c5da9ae4b22d80dadf15a6cb7c09b7e4}] 2023-09-06T20:35:32,869 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure c5da9ae4b22d80dadf15a6cb7c09b7e4 2023-09-06T20:35:33,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36075 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2023-09-06T20:35:33,021 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1694032518189,,1694032518692.c5da9ae4b22d80dadf15a6cb7c09b7e4. 2023-09-06T20:35:33,023 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2819): Flushing c5da9ae4b22d80dadf15a6cb7c09b7e4 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-09-06T20:35:33,041 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/test-1694032518189/c5da9ae4b22d80dadf15a6cb7c09b7e4/.tmp/f/77b0aa523f7241fc92aadb2b01f762f4 is 37, key is row10/f:q1/1694032520864/Put/seqid=0 2023-09-06T20:35:33,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-06T20:35:33,453 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/test-1694032518189/c5da9ae4b22d80dadf15a6cb7c09b7e4/.tmp/f/77b0aa523f7241fc92aadb2b01f762f4 2023-09-06T20:35:33,463 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/test-1694032518189/c5da9ae4b22d80dadf15a6cb7c09b7e4/.tmp/f/77b0aa523f7241fc92aadb2b01f762f4 as hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/test-1694032518189/c5da9ae4b22d80dadf15a6cb7c09b7e4/f/77b0aa523f7241fc92aadb2b01f762f4 2023-09-06T20:35:33,474 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/test-1694032518189/c5da9ae4b22d80dadf15a6cb7c09b7e4/f/77b0aa523f7241fc92aadb2b01f762f4, entries=99, sequenceid=103, filesize=8.2 K 2023-09-06T20:35:33,476 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for c5da9ae4b22d80dadf15a6cb7c09b7e4 in 453ms, sequenceid=103, compaction requested=false 2023-09-06T20:35:33,477 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2520): Flush status journal for c5da9ae4b22d80dadf15a6cb7c09b7e4: 2023-09-06T20:35:33,477 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1694032518189,,1694032518692.c5da9ae4b22d80dadf15a6cb7c09b7e4. for snapshot_1694032532433_default_test-1694032518189 completed. 2023-09-06T20:35:33,478 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(240): Storing 'test-1694032518189,,1694032518692.c5da9ae4b22d80dadf15a6cb7c09b7e4.' region-info for snapshot=snapshot_1694032532433_default_test-1694032518189 2023-09-06T20:35:33,478 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:35:33,478 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/test-1694032518189/c5da9ae4b22d80dadf15a6cb7c09b7e4/f/77b0aa523f7241fc92aadb2b01f762f4] hfiles 2023-09-06T20:35:33,478 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/test-1694032518189/c5da9ae4b22d80dadf15a6cb7c09b7e4/f/77b0aa523f7241fc92aadb2b01f762f4 for snapshot=snapshot_1694032532433_default_test-1694032518189 2023-09-06T20:35:33,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-06T20:35:33,892 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1694032518189,,1694032518692.c5da9ae4b22d80dadf15a6cb7c09b7e4. 2023-09-06T20:35:33,892 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=35 2023-09-06T20:35:33,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.HMaster(3995): Remote procedure done, pid=35 2023-09-06T20:35:33,892 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1694032532433_default_test-1694032518189 on region c5da9ae4b22d80dadf15a6cb7c09b7e4 2023-09-06T20:35:33,894 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure c5da9ae4b22d80dadf15a6cb7c09b7e4 2023-09-06T20:35:33,897 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=35, resume processing ppid=34 2023-09-06T20:35:33,897 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032532433_default_test-1694032518189 table=test-1694032518189 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:35:33,897 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=35, ppid=34, state=SUCCESS; SnapshotRegionProcedure c5da9ae4b22d80dadf15a6cb7c09b7e4 in 1.0290 sec 2023-09-06T20:35:33,898 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032532433_default_test-1694032518189 table=test-1694032518189 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:35:33,898 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032532433_default_test-1694032518189 table=test-1694032518189 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:35:33,898 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1694032532433_default_test-1694032518189 2023-09-06T20:35:33,899 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/.hbase-snapshot/.tmp/snapshot_1694032532433_default_test-1694032518189 2023-09-06T20:35:34,315 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032532433_default_test-1694032518189 table=test-1694032518189 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:35:34,323 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032532433_default_test-1694032518189 table=test-1694032518189 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:35:34,324 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/.hbase-snapshot/.tmp/snapshot_1694032532433_default_test-1694032518189 to hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/.hbase-snapshot/snapshot_1694032532433_default_test-1694032518189 2023-09-06T20:35:34,325 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032532433_default_test-1694032518189 table=test-1694032518189 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:35:34,325 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1694032532433_default_test-1694032518189 table=test-1694032518189 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-09-06T20:35:34,327 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=34, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032532433_default_test-1694032518189 table=test-1694032518189 type=FLUSH ttl=0 } in 1.8780 sec 2023-09-06T20:35:34,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-06T20:35:34,556 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: default:test-1694032518189, procId: 34 completed 2023-09-06T20:35:34,556 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1694032526492 2023-09-06T20:35:34,557 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-09-06T20:35:34,558 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-09-06T20:35:34,561 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1694032526492_test-1694032518189 2023-09-06T20:35:34,561 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1694032532433_default_test-1694032518189 to hdfs://localhost:41733/backupUT/backup_1694032526492/default/test-1694032518189/ 2023-09-06T20:35:34,570 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-09-06T20:35:34,601 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:36869, inputRoot=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd 2023-09-06T20:35:34,601 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,33223,1694032501568_-1202147893_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:41733/backupUT/backup_1694032526492/default/test-1694032518189, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:41733/backupUT/backup_1694032526492/default/test-1694032518189/.hbase-snapshot/.tmp/snapshot_1694032532433_default_test-1694032518189 2023-09-06T20:35:34,601 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:36869, inputRoot=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd, snapshotDir=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/.hbase-snapshot/snapshot_1694032532433_default_test-1694032518189. 2023-09-06T20:35:34,611 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/.hbase-snapshot/snapshot_1694032532433_default_test-1694032518189 to hdfs://localhost:41733/backupUT/backup_1694032526492/default/test-1694032518189/.hbase-snapshot/.tmp/snapshot_1694032532433_default_test-1694032518189 2023-09-06T20:35:35,047 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:35,048 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:35,048 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:35,049 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:36,088 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/hadoop-664337819602863832.jar 2023-09-06T20:35:36,089 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:36,089 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:36,090 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:36,091 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:36,091 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:36,092 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:36,093 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:36,093 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:36,094 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:36,095 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:35:36,096 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:35:36,097 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:35:36,097 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:35:36,098 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:35:36,098 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:35:36,098 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:35:36,099 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:35:36,100 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:35:36,100 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:35:36,101 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:35:36,102 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:35:36,106 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:35:36,106 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:35:36,107 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:35:36,107 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:35:36,108 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:35:36,109 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:35:36,109 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:35:36,171 WARN [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-09-06T20:35:36,200 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1694032532433_default_test-1694032518189' hfile list 2023-09-06T20:35:36,211 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-09-06T20:35:36,513 WARN [Listener at jenkins-hbase3.apache.org/34023 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/mapreduce.cluster.local.dir/1694032536438/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:35:36,514 WARN [Listener at jenkins-hbase3.apache.org/34023 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/mapreduce.cluster.local.dir/1694032536438/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:35:36,712 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-09-06T20:35:36,788 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/test-1694032518189/c5da9ae4b22d80dadf15a6cb7c09b7e4/f/77b0aa523f7241fc92aadb2b01f762f4 output=hdfs://localhost:41733/backupUT/backup_1694032526492/default/test-1694032518189/archive/data/default/test-1694032518189/c5da9ae4b22d80dadf15a6cb7c09b7e4/f/77b0aa523f7241fc92aadb2b01f762f4 2023-09-06T20:35:36,788 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 8.007M/sec 2023-09-06T20:35:37,181 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-09-06T20:35:37,184 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51600, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-09-06T20:35:37,606 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-09-06T20:35:37,607 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-09-06T20:35:37,615 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1694032532433_default_test-1694032518189 2023-09-06T20:35:37,615 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1694032532433_default_test-1694032518189 finished. 2023-09-06T20:35:37,616 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.BackupSystemTable(1014): test-1694032518189 2023-09-06T20:35:37,692 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1694032526492 2023-09-06T20:35:37,692 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-09-06T20:35:38,103 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:41733/backupUT/backup_1694032526492/.backup.manifest 2023-09-06T20:35:38,104 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.TableBackupClient(358): Backup backup_1694032526492 finished: type=FULL,tablelist=test-1694032518189,targetRootDir=hdfs://localhost:41733/backupUT,startts=1694032532145,completets=1694032537689,bytescopied=0 2023-09-06T20:35:38,104 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-09-06T20:35:38,104 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1694032532433_default_test-1694032518189 2023-09-06T20:35:38,110 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1694032532433_default_test-1694032518189" 2023-09-06T20:35:38,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1694032532433_default_test-1694032518189 2023-09-06T20:35:38,115 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1694032532433_default_test-1694032518189 for backup backup_1694032526492 succeeded. 2023-09-06T20:35:38,116 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-09-06T20:35:38,130 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-09-06T20:35:38,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-09-06T20:35:38,133 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-09-06T20:35:38,140 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-09-06T20:35:38,158 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] impl.TableBackupClient(376): Backup backup_1694032526492 completed. 2023-09-06T20:35:38,158 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-06T20:35:38,158 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0ba73ac6 to 127.0.0.1:50423 2023-09-06T20:35:38,158 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:38,158 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] backup.TestRemoteRestore(73): backup complete 2023-09-06T20:35:38,162 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] impl.RestoreTablesClient(101): HBase table table1 does not exist. It will be created during restore process 2023-09-06T20:35:38,162 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41733/backupUT/backup_1694032526492/.backup.manifest 2023-09-06T20:35:38,165 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1694032526492/.backup.manifest 2023-09-06T20:35:38,171 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41733/backupUT/backup_1694032526492/.backup.manifest 2023-09-06T20:35:38,174 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1694032526492/.backup.manifest 2023-09-06T20:35:38,174 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] impl.RestoreTablesClient(148): Restoring 'test-1694032518189' to 'table1' from full backup image hdfs://localhost:41733/backupUT/backup_1694032526492/default/test-1694032518189 2023-09-06T20:35:38,182 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] util.RestoreTool(487): Creating target table 'table1' 2023-09-06T20:35:38,182 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:41733/backupUT/backup_1694032526492/default/test-1694032518189/archive/data/default/test-1694032518189/c5da9ae4b22d80dadf15a6cb7c09b7e4 2023-09-06T20:35:38,183 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:41733/backupUT/backup_1694032526492/default/test-1694032518189/archive/data/default/test-1694032518189/c5da9ae4b22d80dadf15a6cb7c09b7e4/f in region [hdfs://localhost:41733/backupUT/backup_1694032526492/default/test-1694032518189/archive/data/default/test-1694032518189/c5da9ae4b22d80dadf15a6cb7c09b7e4] 2023-09-06T20:35:38,187 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] util.RestoreTool(442): Trying to figure out region boundaries hfile=hdfs://localhost:41733/backupUT/backup_1694032526492/default/test-1694032518189/archive/data/default/test-1694032518189/c5da9ae4b22d80dadf15a6cb7c09b7e4/f/77b0aa523f7241fc92aadb2b01f762f4 first=row0 last=row98 2023-09-06T20:35:38,195 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:35:38,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] procedure2.ProcedureExecutor(1032): Stored pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=table1 2023-09-06T20:35:38,198 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:35:38,198 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 36 2023-09-06T20:35:38,199 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:35:38,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-09-06T20:35:38,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-09-06T20:35:38,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-09-06T20:35:38,607 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => f5facd6c54ed401d7afb02fa219211d1, NAME => 'table1,,1694032538194.f5facd6c54ed401d7afb02fa219211d1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd 2023-09-06T20:35:38,614 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(888): Instantiated table1,,1694032538194.f5facd6c54ed401d7afb02fa219211d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:38,614 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1668): Closing f5facd6c54ed401d7afb02fa219211d1, disabling compactions & flushes 2023-09-06T20:35:38,615 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1690): Closing region table1,,1694032538194.f5facd6c54ed401d7afb02fa219211d1. 2023-09-06T20:35:38,615 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1694032538194.f5facd6c54ed401d7afb02fa219211d1. 2023-09-06T20:35:38,615 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on table1,,1694032538194.f5facd6c54ed401d7afb02fa219211d1. after waiting 0 ms 2023-09-06T20:35:38,615 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region table1,,1694032538194.f5facd6c54ed401d7afb02fa219211d1. 2023-09-06T20:35:38,615 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1909): Closed table1,,1694032538194.f5facd6c54ed401d7afb02fa219211d1. 2023-09-06T20:35:38,615 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1622): Region close journal for f5facd6c54ed401d7afb02fa219211d1: 2023-09-06T20:35:38,616 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:35:38,616 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"table1,,1694032538194.f5facd6c54ed401d7afb02fa219211d1.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1694032538616"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032538616"}]},"ts":"1694032538616"} 2023-09-06T20:35:38,619 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:35:38,620 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:35:38,620 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032538620"}]},"ts":"1694032538620"} 2023-09-06T20:35:38,622 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLING in hbase:meta 2023-09-06T20:35:38,634 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=f5facd6c54ed401d7afb02fa219211d1, ASSIGN}] 2023-09-06T20:35:38,637 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=f5facd6c54ed401d7afb02fa219211d1, ASSIGN 2023-09-06T20:35:38,638 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=table1, region=f5facd6c54ed401d7afb02fa219211d1, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36075,1694032504561; forceNewPlan=false, retain=false 2023-09-06T20:35:38,790 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=f5facd6c54ed401d7afb02fa219211d1, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:38,792 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE; OpenRegionProcedure f5facd6c54ed401d7afb02fa219211d1, server=jenkins-hbase3.apache.org,36075,1694032504561}] 2023-09-06T20:35:38,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-09-06T20:35:38,948 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(130): Open table1,,1694032538194.f5facd6c54ed401d7afb02fa219211d1. 2023-09-06T20:35:38,949 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7254): Opening region: {ENCODED => f5facd6c54ed401d7afb02fa219211d1, NAME => 'table1,,1694032538194.f5facd6c54ed401d7afb02fa219211d1.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:35:38,949 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:35:38,949 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 f5facd6c54ed401d7afb02fa219211d1 2023-09-06T20:35:38,949 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(888): Instantiated table1,,1694032538194.f5facd6c54ed401d7afb02fa219211d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:38,949 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7296): checking encryption for f5facd6c54ed401d7afb02fa219211d1 2023-09-06T20:35:38,949 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7299): checking classloading for f5facd6c54ed401d7afb02fa219211d1 2023-09-06T20:35:38,951 INFO [StoreOpener-f5facd6c54ed401d7afb02fa219211d1-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region f5facd6c54ed401d7afb02fa219211d1 2023-09-06T20:35:38,952 INFO [StoreOpener-f5facd6c54ed401d7afb02fa219211d1-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f5facd6c54ed401d7afb02fa219211d1 columnFamilyName f 2023-09-06T20:35:38,952 DEBUG [StoreOpener-f5facd6c54ed401d7afb02fa219211d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:38,953 INFO [StoreOpener-f5facd6c54ed401d7afb02fa219211d1-1 {}] regionserver.HStore(324): Store=f5facd6c54ed401d7afb02fa219211d1/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:35:38,953 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/table1/f5facd6c54ed401d7afb02fa219211d1 2023-09-06T20:35:38,954 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/table1/f5facd6c54ed401d7afb02fa219211d1 2023-09-06T20:35:38,956 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1076): writing seq id for f5facd6c54ed401d7afb02fa219211d1 2023-09-06T20:35:38,959 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/table1/f5facd6c54ed401d7afb02fa219211d1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:35:38,960 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): Opened f5facd6c54ed401d7afb02fa219211d1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11442744800, jitterRate=0.06568865478038788}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:35:38,961 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(992): Region open journal for f5facd6c54ed401d7afb02fa219211d1: 2023-09-06T20:35:38,962 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2556): Post open deploy tasks for table1,,1694032538194.f5facd6c54ed401d7afb02fa219211d1., pid=38, masterSystemTime=1694032538944 2023-09-06T20:35:38,963 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2583): Finished post open deploy task for table1,,1694032538194.f5facd6c54ed401d7afb02fa219211d1. 2023-09-06T20:35:38,963 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(158): Opened table1,,1694032538194.f5facd6c54ed401d7afb02fa219211d1. 2023-09-06T20:35:38,964 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=f5facd6c54ed401d7afb02fa219211d1, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:38,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=38, resume processing ppid=37 2023-09-06T20:35:38,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=38, ppid=37, state=SUCCESS; OpenRegionProcedure f5facd6c54ed401d7afb02fa219211d1, server=jenkins-hbase3.apache.org,36075,1694032504561 in 174 msec 2023-09-06T20:35:38,969 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=37, resume processing ppid=36 2023-09-06T20:35:38,970 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=37, ppid=36, state=SUCCESS; TransitRegionStateProcedure table=table1, region=f5facd6c54ed401d7afb02fa219211d1, ASSIGN in 333 msec 2023-09-06T20:35:38,970 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:35:38,970 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032538970"}]},"ts":"1694032538970"} 2023-09-06T20:35:38,972 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLED in hbase:meta 2023-09-06T20:35:38,974 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:35:38,975 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=36, state=SUCCESS; CreateTableProcedure table=table1 in 779 msec 2023-09-06T20:35:39,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-09-06T20:35:39,304 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:table1, procId: 36 completed 2023-09-06T20:35:39,310 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:41733/backupUT/backup_1694032526492/default/test-1694032518189/archive/data/default/test-1694032518189/c5da9ae4b22d80dadf15a6cb7c09b7e4 from hbase tables test-1694032518189 to tables table1 2023-09-06T20:35:39,310 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.MapReduceRestoreJob(72): Restore test-1694032518189 into table1 2023-09-06T20:35:39,313 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/hbase-staging/bulk_output-default-table1-1694032539310 from hdfs://localhost:41733/backupUT/backup_1694032526492/default/test-1694032518189/archive/data/default/test-1694032518189/c5da9ae4b22d80dadf15a6cb7c09b7e4 2023-09-06T20:35:39,317 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0fb4f79b to 127.0.0.1:50423 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:35:39,324 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63a8a2dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:39,326 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:35:39,327 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51614, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:35:39,344 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-09-06T20:35:39,344 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table table1 2023-09-06T20:35:39,350 DEBUG [hconnection-0x75525c3f-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:39,352 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49022, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:39,355 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for table1: 2023-09-06T20:35:39,356 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-09-06T20:35:39,356 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/hbase-staging/partitions_6c967371-ca14-41b1-b72c-e207e2cbecfa 2023-09-06T20:35:39,805 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:39,805 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:39,806 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:39,806 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:40,804 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/hadoop-6738014874572909545.jar 2023-09-06T20:35:40,804 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:40,804 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:40,805 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:40,805 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:40,806 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:40,806 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:40,807 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:40,807 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:40,807 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:40,808 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:35:40,809 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:35:40,809 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:35:40,810 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:35:40,810 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:35:40,811 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:35:40,811 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:35:40,811 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:35:40,812 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:35:40,812 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:35:40,813 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:35:40,813 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:35:40,817 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:40,817 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:40,818 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:40,818 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:40,818 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:40,819 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:40,819 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:35:40,819 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: table1 2023-09-06T20:35:40,820 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-06T20:35:40,820 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0fb4f79b to 127.0.0.1:50423 2023-09-06T20:35:40,820 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:40,820 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-09-06T20:35:40,820 WARN [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-09-06T20:35:40,821 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:35:40,826 WARN [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-09-06T20:35:40,833 WARN [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-09-06T20:35:40,925 WARN [Listener at jenkins-hbase3.apache.org/34023 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/mapreduce.cluster.local.dir/1694032540885/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:35:40,925 WARN [Listener at jenkins-hbase3.apache.org/34023 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/mapreduce.cluster.local.dir/1694032540885/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:35:40,992 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:41733/backupUT/backup_1694032526492/default/test-1694032518189/archive/data/default/test-1694032518189/c5da9ae4b22d80dadf15a6cb7c09b7e4/f/77b0aa523f7241fc92aadb2b01f762f4 2023-09-06T20:35:40,997 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-09-06T20:35:41,121 DEBUG [pool-322-thread-1 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2fdca36c to 127.0.0.1:50423 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:35:41,129 DEBUG [pool-322-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ba66e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:41,131 DEBUG [hconnection-0x58e96ddf-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:41,133 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:39432, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:41,134 DEBUG [pool-322-thread-1 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2fdca36c to 127.0.0.1:50423 2023-09-06T20:35:41,135 DEBUG [pool-322-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:41,137 DEBUG [pool-322-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-09-06T20:35:41,137 DEBUG [pool-322-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-09-06T20:35:41,152 DEBUG [pool-322-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/hbase-staging/bulk_output-default-table1-1694032539310/_temporary/0/_temporary/attempt_local897900390_0002_r_000000_0/f/e82897af5cd1492f87471a2d4e2bcbb3 is 37, key is row10/f:q1/1694032520864/Put/seqid=0 2023-09-06T20:35:41,154 WARN [Thread-968 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:36075] Specified favored nodes: [jenkins-hbase3.apache.org:36075] 2023-09-06T20:35:41,982 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/hbase-staging/bulk_output-default-table1-1694032539310 2023-09-06T20:35:41,983 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0bbb7db0 to 127.0.0.1:50423 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:35:41,989 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61cbdcea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:41,990 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:41,992 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:39438, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:42,001 WARN [Listener at jenkins-hbase3.apache.org/34023 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/hbase-staging/bulk_output-default-table1-1694032539310/_SUCCESS 2023-09-06T20:35:42,006 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:35:42,008 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:34704, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:35:42,039 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/hbase-staging/bulk_output-default-table1-1694032539310/f/e82897af5cd1492f87471a2d4e2bcbb3 first=Optional[row0] last=Optional[row98] 2023-09-06T20:35:42,058 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=table1,,1694032538194.f5facd6c54ed401d7afb02fa219211d1., hostname=jenkins-hbase3.apache.org,36075,1694032504561, seqNum=2 for row with hfile group [{f,hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/hbase-staging/bulk_output-default-table1-1694032539310/f/e82897af5cd1492f87471a2d4e2bcbb3}] 2023-09-06T20:35:42,079 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/hbase-staging/bulk_output-default-table1-1694032539310/f/e82897af5cd1492f87471a2d4e2bcbb3 for inclusion in f5facd6c54ed401d7afb02fa219211d1/f 2023-09-06T20:35:42,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-09-06T20:35:42,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] regionserver.HStore(625): Region bounds: first= last= 2023-09-06T20:35:42,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] regionserver.HRegion(2520): Flush status journal for f5facd6c54ed401d7afb02fa219211d1: 2023-09-06T20:35:42,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/hbase-staging/bulk_output-default-table1-1694032539310/f/e82897af5cd1492f87471a2d4e2bcbb3 to hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/staging/jenkins__table1__rff0i46ru4lc91qqnjf7cqmdofiklrh9sigprqde1i4v8a5paepuanam20k80jk3/f/e82897af5cd1492f87471a2d4e2bcbb3 2023-09-06T20:35:42,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/staging/jenkins__table1__rff0i46ru4lc91qqnjf7cqmdofiklrh9sigprqde1i4v8a5paepuanam20k80jk3/f/e82897af5cd1492f87471a2d4e2bcbb3 as hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/table1/f5facd6c54ed401d7afb02fa219211d1/f/43dad1b6b7f14d02879e39e66612cfe4_SeqId_4_ 2023-09-06T20:35:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3219874d to 127.0.0.1:50423 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:35:42,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@223ec367, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:42,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:35:42,106 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:34714, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:35:42,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:42,111 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:39440, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:42,129 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-06T20:35:42,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3219874d to 127.0.0.1:50423 2023-09-06T20:35:42,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:42,130 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/staging/jenkins__table1__rff0i46ru4lc91qqnjf7cqmdofiklrh9sigprqde1i4v8a5paepuanam20k80jk3/f/e82897af5cd1492f87471a2d4e2bcbb3 into f5facd6c54ed401d7afb02fa219211d1/f as hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/table1/f5facd6c54ed401d7afb02fa219211d1/f/43dad1b6b7f14d02879e39e66612cfe4_SeqId_4_ - updating store file list. 2023-09-06T20:35:42,137 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/table1/f5facd6c54ed401d7afb02fa219211d1/f/43dad1b6b7f14d02879e39e66612cfe4_SeqId_4_ into f5facd6c54ed401d7afb02fa219211d1/f 2023-09-06T20:35:42,137 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/staging/jenkins__table1__rff0i46ru4lc91qqnjf7cqmdofiklrh9sigprqde1i4v8a5paepuanam20k80jk3/f/e82897af5cd1492f87471a2d4e2bcbb3 into f5facd6c54ed401d7afb02fa219211d1/f (new location: hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/table1/f5facd6c54ed401d7afb02fa219211d1/f/43dad1b6b7f14d02879e39e66612cfe4_SeqId_4_) 2023-09-06T20:35:42,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/staging/jenkins__table1__rff0i46ru4lc91qqnjf7cqmdofiklrh9sigprqde1i4v8a5paepuanam20k80jk3/f/e82897af5cd1492f87471a2d4e2bcbb3 2023-09-06T20:35:42,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7524c7e2 to 127.0.0.1:50423 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:35:42,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c77f8ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:42,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:35:42,161 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:34728, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-09-06T20:35:42,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:42,166 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:39456, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-09-06T20:35:42,180 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-06T20:35:42,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7524c7e2 to 127.0.0.1:50423 2023-09-06T20:35:42,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:42,192 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-06T20:35:42,192 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0bbb7db0 to 127.0.0.1:50423 2023-09-06T20:35:42,193 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:42,193 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-09-06T20:35:42,193 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-09-06T20:35:42,193 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] impl.RestoreTablesClient(233): Backup: backup_1694032526492 hdfs://localhost:41733/backupUT/backup_1694032526492/default/test-1694032518189/ 2023-09-06T20:35:42,193 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-09-06T20:35:42,195 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] client.HBaseAdmin$18(967): Started disable of table1 2023-09-06T20:35:42,201 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable table1 2023-09-06T20:35:42,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] procedure2.ProcedureExecutor(1032): Stored pid=39, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=table1 2023-09-06T20:35:42,217 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032542217"}]},"ts":"1694032542217"} 2023-09-06T20:35:42,219 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLING in hbase:meta 2023-09-06T20:35:42,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=39 2023-09-06T20:35:42,221 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(293): Set table1 to state=DISABLING 2023-09-06T20:35:42,223 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=f5facd6c54ed401d7afb02fa219211d1, UNASSIGN}] 2023-09-06T20:35:42,226 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=f5facd6c54ed401d7afb02fa219211d1, UNASSIGN 2023-09-06T20:35:42,227 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=f5facd6c54ed401d7afb02fa219211d1, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:42,228 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE; CloseRegionProcedure f5facd6c54ed401d7afb02fa219211d1, server=jenkins-hbase3.apache.org,36075,1694032504561}] 2023-09-06T20:35:42,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=39 2023-09-06T20:35:42,389 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] handler.UnassignRegionHandler(119): Close f5facd6c54ed401d7afb02fa219211d1 2023-09-06T20:35:42,389 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1668): Closing f5facd6c54ed401d7afb02fa219211d1, disabling compactions & flushes 2023-09-06T20:35:42,390 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1690): Closing region table1,,1694032538194.f5facd6c54ed401d7afb02fa219211d1. 2023-09-06T20:35:42,390 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1694032538194.f5facd6c54ed401d7afb02fa219211d1. 2023-09-06T20:35:42,390 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1778): Acquired close lock on table1,,1694032538194.f5facd6c54ed401d7afb02fa219211d1. after waiting 0 ms 2023-09-06T20:35:42,390 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1788): Updates disabled for region table1,,1694032538194.f5facd6c54ed401d7afb02fa219211d1. 2023-09-06T20:35:42,398 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/table1/f5facd6c54ed401d7afb02fa219211d1/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-09-06T20:35:42,399 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:35:42,399 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1909): Closed table1,,1694032538194.f5facd6c54ed401d7afb02fa219211d1. 2023-09-06T20:35:42,399 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1622): Region close journal for f5facd6c54ed401d7afb02fa219211d1: 2023-09-06T20:35:42,400 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] handler.UnassignRegionHandler(163): Closed f5facd6c54ed401d7afb02fa219211d1 2023-09-06T20:35:42,401 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=f5facd6c54ed401d7afb02fa219211d1, regionState=CLOSED 2023-09-06T20:35:42,404 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=41, resume processing ppid=40 2023-09-06T20:35:42,404 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=41, ppid=40, state=SUCCESS; CloseRegionProcedure f5facd6c54ed401d7afb02fa219211d1, server=jenkins-hbase3.apache.org,36075,1694032504561 in 174 msec 2023-09-06T20:35:42,406 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=40, resume processing ppid=39 2023-09-06T20:35:42,406 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=40, ppid=39, state=SUCCESS; TransitRegionStateProcedure table=table1, region=f5facd6c54ed401d7afb02fa219211d1, UNASSIGN in 181 msec 2023-09-06T20:35:42,407 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032542407"}]},"ts":"1694032542407"} 2023-09-06T20:35:42,408 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLED in hbase:meta 2023-09-06T20:35:42,411 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(305): Set table1 to state=DISABLED 2023-09-06T20:35:42,413 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=39, state=SUCCESS; DisableTableProcedure table=table1 in 207 msec 2023-09-06T20:35:42,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=39 2023-09-06T20:35:42,522 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] client.HBaseAdmin$TableFuture(3677): Operation: DISABLE, Table Name: default:table1, procId: 39 completed 2023-09-06T20:35:42,527 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.HMaster$5(2448): Client=jenkins//172.31.12.81 delete table1 2023-09-06T20:35:42,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] procedure2.ProcedureExecutor(1032): Stored pid=42, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=table1 2023-09-06T20:35:42,537 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=42, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-09-06T20:35:42,539 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=42, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=table1 2023-09-06T20:35:42,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-09-06T20:35:42,545 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/table1/f5facd6c54ed401d7afb02fa219211d1 2023-09-06T20:35:42,549 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/table1/f5facd6c54ed401d7afb02fa219211d1/f, FileablePath, hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/table1/f5facd6c54ed401d7afb02fa219211d1/recovered.edits] 2023-09-06T20:35:42,557 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/table1/f5facd6c54ed401d7afb02fa219211d1/f/43dad1b6b7f14d02879e39e66612cfe4_SeqId_4_ to hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/archive/data/default/table1/f5facd6c54ed401d7afb02fa219211d1/f/43dad1b6b7f14d02879e39e66612cfe4_SeqId_4_ 2023-09-06T20:35:42,560 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/table1/f5facd6c54ed401d7afb02fa219211d1/recovered.edits/6.seqid to hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/archive/data/default/table1/f5facd6c54ed401d7afb02fa219211d1/recovered.edits/6.seqid 2023-09-06T20:35:42,561 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/table1/f5facd6c54ed401d7afb02fa219211d1 2023-09-06T20:35:42,561 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(293): Archived table1 regions 2023-09-06T20:35:42,563 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=42, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=table1 2023-09-06T20:35:42,572 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of table1 from hbase:meta 2023-09-06T20:35:42,575 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(387): Removing 'table1' descriptor. 2023-09-06T20:35:42,576 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=42, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=table1 2023-09-06T20:35:42,576 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(377): Removing 'table1' from region states. 2023-09-06T20:35:42,576 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1,,1694032538194.f5facd6c54ed401d7afb02fa219211d1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1694032542576"}]},"ts":"9223372036854775807"} 2023-09-06T20:35:42,578 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-06T20:35:42,578 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => f5facd6c54ed401d7afb02fa219211d1, NAME => 'table1,,1694032538194.f5facd6c54ed401d7afb02fa219211d1.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:35:42,578 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(381): Marking 'table1' as deleted. 2023-09-06T20:35:42,578 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1694032542578"}]},"ts":"9223372036854775807"} 2023-09-06T20:35:42,580 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1678): Deleted table table1 state from META 2023-09-06T20:35:42,582 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(127): Finished pid=42, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-09-06T20:35:42,583 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=42, state=SUCCESS; DeleteTableProcedure table=table1 in 53 msec 2023-09-06T20:35:42,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-09-06T20:35:42,642 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] client.HBaseAdmin$TableFuture(3677): Operation: DELETE, Table Name: default:table1, procId: 42 completed 2023-09-06T20:35:42,667 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.ResourceChecker(175): after: backup.TestRemoteRestore#testFullRestoreRemote Thread=830 (was 815) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1ea85617-shared-pool-11 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: member: 'jenkins-hbase3.apache.org,36075,1694032504561' subprocedure-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1ea85617-shared-pool-10 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: HFileArchiver-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1ea85617-shared-pool-7 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-91703826_16 at /127.0.0.1:47492 [Receiving block BP-657318226-172.31.12.81-1694032498405:blk_1073741855_1031] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1ea85617-shared-pool-9 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/cluster_8944b83c-4af0-4d3a-395e-af5ae00b91a1/dfs/data/data2/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/cluster_8944b83c-4af0-4d3a-395e-af5ae00b91a1/dfs/data/data1/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Client (366321005) connection to localhost/127.0.0.1:41733 from jenkins.hfs.1 java.lang.Object.wait(Native Method) org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1053) org.apache.hadoop.ipc.Client$Connection.run(Client.java:1097) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-91703826_16 at /127.0.0.1:47494 [Receiving block BP-657318226-172.31.12.81-1694032498405:blk_1073741856_1032] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1ea85617-shared-pool-6 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-657318226-172.31.12.81-1694032498405:blk_1073741855_1031, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1118506683_16 at /127.0.0.1:45088 [Waiting for operation #6] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1ea85617-shared-pool-8 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_116006268_16 at /127.0.0.1:50046 [Waiting for operation #8] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-657318226-172.31.12.81-1694032498405:blk_1073741856_1032, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,36075,1694032504561)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: (jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=905 (was 881) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=139 (was 161), ProcessCount=167 (was 167), AvailableMemoryMB=3458 (was 4025) 2023-09-06T20:35:42,667 WARN [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.ResourceChecker(130): Thread=830 is superior to 500 2023-09-06T20:35:42,690 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.ResourceChecker(147): before: backup.TestRemoteRestore#testFullRestoreRemoteWithAlternateRestoreOutputDir Thread=830, OpenFileDescriptor=905, MaxFileDescriptor=60000, SystemLoadAverage=139, ProcessCount=167, AvailableMemoryMB=3457 2023-09-06T20:35:42,690 WARN [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.ResourceChecker(130): Thread=830 is superior to 500 2023-09-06T20:35:42,690 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] backup.TestRemoteRestore(91): test remote full backup on a single table with alternate restore output dir 2023-09-06T20:35:42,691 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3754a699 to 127.0.0.1:50423 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:35:42,698 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77c02e3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:42,701 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:42,703 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:39470, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:42,704 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:35:42,706 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:34730, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:35:42,720 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-09-06T20:35:42,725 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:35:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032542726 (current time:1694032542726). 2023-09-06T20:35:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:35:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-09-06T20:35:42,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x65cbfb2e to 127.0.0.1:50423 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:35:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@303acb0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:42,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:42,735 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:39472, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x65cbfb2e to 127.0.0.1:50423 2023-09-06T20:35:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:42,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:35:42,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] procedure2.ProcedureExecutor(1032): Stored pid=43, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:35:42,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 43 2023-09-06T20:35:42,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=43 2023-09-06T20:35:42,739 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:35:42,740 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:35:42,742 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:35:42,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=43 2023-09-06T20:35:43,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=43 2023-09-06T20:35:43,149 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:35:43,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE; SnapshotRegionProcedure 8e818fe31858c9c4a15e134b8245804a}] 2023-09-06T20:35:43,151 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=44, ppid=43, state=RUNNABLE; SnapshotRegionProcedure 8e818fe31858c9c4a15e134b8245804a 2023-09-06T20:35:43,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36075 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=44 2023-09-06T20:35:43,303 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1694032526663.8e818fe31858c9c4a15e134b8245804a. 2023-09-06T20:35:43,304 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegion(2819): Flushing 8e818fe31858c9c4a15e134b8245804a 2/2 column families, dataSize=1.08 KB heapSize=2.23 KB 2023-09-06T20:35:43,318 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/.tmp/meta/cd5c9a1a21474c60b2ff477d2560ff93 is 169, key is trslm:hdfs://localhost:41733/backupUT\x00test-1694032518189/meta:log-roll-map/1694032537677/Put/seqid=0 2023-09-06T20:35:43,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=43 2023-09-06T20:35:43,725 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=526 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/.tmp/meta/cd5c9a1a21474c60b2ff477d2560ff93 2023-09-06T20:35:43,734 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/.tmp/session/dba4ee12858d401b94265db4b05495cd is 310, key is session:backup_1694032526492/session:context/1694032538139/Put/seqid=0 2023-09-06T20:35:43,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=43 2023-09-06T20:35:44,139 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=579 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/.tmp/session/dba4ee12858d401b94265db4b05495cd 2023-09-06T20:35:44,146 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/.tmp/meta/cd5c9a1a21474c60b2ff477d2560ff93 as hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/meta/cd5c9a1a21474c60b2ff477d2560ff93 2023-09-06T20:35:44,152 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/meta/cd5c9a1a21474c60b2ff477d2560ff93, entries=4, sequenceid=17, filesize=5.5 K 2023-09-06T20:35:44,153 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/.tmp/session/dba4ee12858d401b94265db4b05495cd as hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/session/dba4ee12858d401b94265db4b05495cd 2023-09-06T20:35:44,160 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/session/dba4ee12858d401b94265db4b05495cd, entries=2, sequenceid=17, filesize=5.3 K 2023-09-06T20:35:44,161 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegion(3022): Finished flush of dataSize ~1.08 KB/1105, heapSize ~2.20 KB/2256, currentSize=0 B/0 for 8e818fe31858c9c4a15e134b8245804a in 858ms, sequenceid=17, compaction requested=false 2023-09-06T20:35:44,161 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegion(2520): Flush status journal for 8e818fe31858c9c4a15e134b8245804a: 2023-09-06T20:35:44,161 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1694032526663.8e818fe31858c9c4a15e134b8245804a. for snapshot_backup_system completed. 2023-09-06T20:35:44,161 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1694032526663.8e818fe31858c9c4a15e134b8245804a.' region-info for snapshot=snapshot_backup_system 2023-09-06T20:35:44,161 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:35:44,161 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/meta/cd5c9a1a21474c60b2ff477d2560ff93] hfiles 2023-09-06T20:35:44,161 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/meta/cd5c9a1a21474c60b2ff477d2560ff93 for snapshot=snapshot_backup_system 2023-09-06T20:35:44,161 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/session/db85f23ebe2a4f8b9aaa70d53427ac0d, hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/session/dba4ee12858d401b94265db4b05495cd] hfiles 2023-09-06T20:35:44,161 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/session/db85f23ebe2a4f8b9aaa70d53427ac0d for snapshot=snapshot_backup_system 2023-09-06T20:35:44,161 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/session/dba4ee12858d401b94265db4b05495cd for snapshot=snapshot_backup_system 2023-09-06T20:35:44,551 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:35:44,568 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1694032526663.8e818fe31858c9c4a15e134b8245804a. 2023-09-06T20:35:44,568 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=44 2023-09-06T20:35:44,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.HMaster(3995): Remote procedure done, pid=44 2023-09-06T20:35:44,568 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 8e818fe31858c9c4a15e134b8245804a 2023-09-06T20:35:44,570 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=44, ppid=43, state=RUNNABLE; SnapshotRegionProcedure 8e818fe31858c9c4a15e134b8245804a 2023-09-06T20:35:44,572 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=44, resume processing ppid=43 2023-09-06T20:35:44,572 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=44, ppid=43, state=SUCCESS; SnapshotRegionProcedure 8e818fe31858c9c4a15e134b8245804a in 1.4220 sec 2023-09-06T20:35:44,572 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:35:44,573 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:35:44,574 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:35:44,574 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-09-06T20:35:44,575 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/.hbase-snapshot/.tmp/snapshot_backup_system 2023-09-06T20:35:44,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=43 2023-09-06T20:35:45,006 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:35:45,013 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:35:45,014 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/.hbase-snapshot/snapshot_backup_system 2023-09-06T20:35:45,015 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:35:45,015 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 43 2023-09-06T20:35:45,016 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=43, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2780 sec 2023-09-06T20:35:46,299 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-09-06T20:35:46,302 INFO [RS-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42228, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2023-09-06T20:35:46,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=43 2023-09-06T20:35:46,846 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: backup:system, procId: 43 completed 2023-09-06T20:35:46,846 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] impl.TableBackupClient(115): Backup backup_1694032542698 started at 1694032546846. 2023-09-06T20:35:46,848 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.TableBackupClient(119): Backup session backup_1694032542698 has been started. 2023-09-06T20:35:46,854 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-09-06T20:35:46,855 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-09-06T20:35:46,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-09-06T20:35:46,855 INFO [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-09-06T20:35:46,855 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-09-06T20:35:46,856 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-09-06T20:35:46,856 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-09-06T20:35:46,857 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-09-06T20:35:46,857 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:35:46,858 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:46,858 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-09-06T20:35:46,858 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:35:46,858 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:35:46,858 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:46,858 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-09-06T20:35:46,858 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-09-06T20:35:46,859 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-09-06T20:35:46,859 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-09-06T20:35:46,859 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:35:46,859 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-09-06T20:35:46,859 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-09-06T20:35:46,859 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-09-06T20:35:46,859 DEBUG [member: 'jenkins-hbase3.apache.org,36075,1694032504561' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-09-06T20:35:46,860 DEBUG [member: 'jenkins-hbase3.apache.org,36075,1694032504561' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-09-06T20:35:46,860 DEBUG [member: 'jenkins-hbase3.apache.org,36075,1694032504561' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-09-06T20:35:46,860 DEBUG [member: 'jenkins-hbase3.apache.org,36075,1694032504561' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-09-06T20:35:46,861 DEBUG [member: 'jenkins-hbase3.apache.org,36075,1694032504561' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,36075,1694032504561' joining acquired barrier for procedure (rolllog) in zk 2023-09-06T20:35:46,862 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:46,862 DEBUG [member: 'jenkins-hbase3.apache.org,36075,1694032504561' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-09-06T20:35:46,862 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:46,862 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:35:46,862 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:35:46,862 DEBUG [member: 'jenkins-hbase3.apache.org,36075,1694032504561' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-09-06T20:35:46,862 DEBUG [member: 'jenkins-hbase3.apache.org,36075,1694032504561' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-09-06T20:35:46,862 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:35:46,863 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:35:46,863 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:35:46,863 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:46,863 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:35:46,864 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,36075,1694032504561' joining acquired barrier for procedure 'rolllog' on coordinator 2023-09-06T20:35:46,864 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@5561cc75[Count = 0] remaining members to acquire global barrier 2023-09-06T20:35:46,864 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-09-06T20:35:46,864 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-09-06T20:35:46,865 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-09-06T20:35:46,865 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-09-06T20:35:46,865 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-09-06T20:35:46,865 DEBUG [member: 'jenkins-hbase3.apache.org,36075,1694032504561' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-09-06T20:35:46,865 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:46,865 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-09-06T20:35:46,865 DEBUG [member: 'jenkins-hbase3.apache.org,36075,1694032504561' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-09-06T20:35:46,865 DEBUG [rs(jenkins-hbase3.apache.org,36075,1694032504561)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:46,865 INFO [rs(jenkins-hbase3.apache.org,36075,1694032504561)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1694032532222 highest: 1694032532222 on jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:46,865 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36075%2C1694032504561:(num 1694032532222) roll requested 2023-09-06T20:35:46,905 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/WALs/jenkins-hbase3.apache.org,36075,1694032504561/jenkins-hbase3.apache.org%2C36075%2C1694032504561.1694032546866, exclude list is [], retry=0 2023-09-06T20:35:46,910 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46843,DS-1307a321-484a-4087-acdc-931715071dc3,DISK] 2023-09-06T20:35:46,911 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/WALs/jenkins-hbase3.apache.org,36075,1694032504561/jenkins-hbase3.apache.org%2C36075%2C1694032504561.1694032532222 with entries=15, filesize=3.47 KB; new WAL /user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/WALs/jenkins-hbase3.apache.org,36075,1694032504561/jenkins-hbase3.apache.org%2C36075%2C1694032504561.1694032546866 2023-09-06T20:35:46,911 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:46843,DS-1307a321-484a-4087-acdc-931715071dc3,DISK]] 2023-09-06T20:35:46,911 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/WALs/jenkins-hbase3.apache.org,36075,1694032504561/jenkins-hbase3.apache.org%2C36075%2C1694032504561.1694032532222 is not closed yet, will try archiving it next time 2023-09-06T20:35:46,911 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36075%2C1694032504561.meta:.meta(num 1694032532257) roll requested 2023-09-06T20:35:46,924 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/WALs/jenkins-hbase3.apache.org,36075,1694032504561/jenkins-hbase3.apache.org%2C36075%2C1694032504561.meta.1694032546912.meta, exclude list is [], retry=0 2023-09-06T20:35:46,930 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46843,DS-1307a321-484a-4087-acdc-931715071dc3,DISK] 2023-09-06T20:35:46,931 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/WALs/jenkins-hbase3.apache.org,36075,1694032504561/jenkins-hbase3.apache.org%2C36075%2C1694032504561.meta.1694032532257.meta with entries=12, filesize=2.73 KB; new WAL /user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/WALs/jenkins-hbase3.apache.org,36075,1694032504561/jenkins-hbase3.apache.org%2C36075%2C1694032504561.meta.1694032546912.meta 2023-09-06T20:35:46,931 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:46843,DS-1307a321-484a-4087-acdc-931715071dc3,DISK]] 2023-09-06T20:35:46,932 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/WALs/jenkins-hbase3.apache.org,36075,1694032504561/jenkins-hbase3.apache.org%2C36075%2C1694032504561.meta.1694032532257.meta is not closed yet, will try archiving it next time 2023-09-06T20:35:46,934 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/WALs/jenkins-hbase3.apache.org,36075,1694032504561/jenkins-hbase3.apache.org%2C36075%2C1694032504561.meta.1694032532257.meta not finished, retry = 0 2023-09-06T20:35:46,943 DEBUG [rs(jenkins-hbase3.apache.org,36075,1694032504561)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 78 2023-09-06T20:35:46,943 INFO [rs(jenkins-hbase3.apache.org,36075,1694032504561)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1694032546866 on jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:46,952 DEBUG [member: 'jenkins-hbase3.apache.org,36075,1694032504561' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-09-06T20:35:46,952 DEBUG [member: 'jenkins-hbase3.apache.org,36075,1694032504561' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,36075,1694032504561' in zk 2023-09-06T20:35:46,955 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:46,955 DEBUG [member: 'jenkins-hbase3.apache.org,36075,1694032504561' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-09-06T20:35:46,955 DEBUG [member: 'jenkins-hbase3.apache.org,36075,1694032504561' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:35:46,955 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:46,955 DEBUG [member: 'jenkins-hbase3.apache.org,36075,1694032504561' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-09-06T20:35:46,956 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:35:46,956 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:35:46,957 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:35:46,957 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:35:46,957 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:35:46,958 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:46,958 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:35:46,958 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:35:46,958 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:46,959 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,36075,1694032504561': 2023-09-06T20:35:46,959 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,36075,1694032504561' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-09-06T20:35:46,959 INFO [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-09-06T20:35:46,959 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-09-06T20:35:46,959 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-09-06T20:35:46,959 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-09-06T20:35:46,959 INFO [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-09-06T20:35:46,961 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-09-06T20:35:46,961 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-09-06T20:35:46,961 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-09-06T20:35:46,961 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-09-06T20:35:46,961 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-09-06T20:35:46,961 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:35:46,961 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:35:46,961 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:35:46,961 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:35:46,961 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:35:46,961 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:35:46,961 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:46,962 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:35:46,962 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:35:46,962 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:35:46,962 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:35:46,962 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:46,962 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:46,963 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:35:46,963 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:35:46,963 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:46,965 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:46,965 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-09-06T20:35:46,965 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-09-06T20:35:46,965 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-09-06T20:35:46,965 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-09-06T20:35:46,965 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:35:46,965 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:46,965 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:35:46,965 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-09-06T20:35:46,965 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-09-06T20:35:46,965 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-09-06T20:35:46,965 DEBUG [(jenkins-hbase3.apache.org,33223,1694032501568)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:35:46,965 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-09-06T20:35:46,966 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-09-06T20:35:46,966 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:35:46,966 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:35:46,967 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] client.HBaseAdmin(2838): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-09-06T20:35:46,967 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] client.HBaseAdmin(2846): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-09-06T20:35:47,067 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] client.HBaseAdmin(2852): Getting current status of procedure from master... 2023-09-06T20:35:47,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-09-06T20:35:47,071 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1694032547070_default_test-1694032518189 table=test-1694032518189 type=FLUSH ttl=0 } 2023-09-06T20:35:47,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032547071 (current time:1694032547071). 2023-09-06T20:35:47,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:35:47,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1694032547070_default_test-1694032518189 VERSION not specified, setting to 2 2023-09-06T20:35:47,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0d645d55 to 127.0.0.1:50423 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:35:47,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15e8ee34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:47,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:47,090 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:39476, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:47,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0d645d55 to 127.0.0.1:50423 2023-09-06T20:35:47,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:47,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:35:47,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] procedure2.ProcedureExecutor(1032): Stored pid=45, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1694032547070_default_test-1694032518189 table=test-1694032518189 type=FLUSH ttl=0 } 2023-09-06T20:35:47,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1694032547070_default_test-1694032518189 table=test-1694032518189 type=FLUSH ttl=0 }, snapshot procedure id = 45 2023-09-06T20:35:47,095 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1694032547070_default_test-1694032518189 table=test-1694032518189 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:35:47,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=45 2023-09-06T20:35:47,096 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1694032547070_default_test-1694032518189 table=test-1694032518189 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:35:47,098 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1694032547070_default_test-1694032518189 table=test-1694032518189 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:35:47,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=45 2023-09-06T20:35:47,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=45 2023-09-06T20:35:47,505 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1694032547070_default_test-1694032518189 table=test-1694032518189 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:35:47,505 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE; SnapshotRegionProcedure c5da9ae4b22d80dadf15a6cb7c09b7e4}] 2023-09-06T20:35:47,508 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=46, ppid=45, state=RUNNABLE; SnapshotRegionProcedure c5da9ae4b22d80dadf15a6cb7c09b7e4 2023-09-06T20:35:47,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36075 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=46 2023-09-06T20:35:47,659 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1694032518189,,1694032518692.c5da9ae4b22d80dadf15a6cb7c09b7e4. 2023-09-06T20:35:47,660 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] regionserver.HRegion(2520): Flush status journal for c5da9ae4b22d80dadf15a6cb7c09b7e4: 2023-09-06T20:35:47,660 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1694032518189,,1694032518692.c5da9ae4b22d80dadf15a6cb7c09b7e4. for snapshot_1694032547070_default_test-1694032518189 completed. 2023-09-06T20:35:47,660 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] snapshot.SnapshotManifest(240): Storing 'test-1694032518189,,1694032518692.c5da9ae4b22d80dadf15a6cb7c09b7e4.' region-info for snapshot=snapshot_1694032547070_default_test-1694032518189 2023-09-06T20:35:47,660 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:35:47,660 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/test-1694032518189/c5da9ae4b22d80dadf15a6cb7c09b7e4/f/77b0aa523f7241fc92aadb2b01f762f4] hfiles 2023-09-06T20:35:47,660 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/test-1694032518189/c5da9ae4b22d80dadf15a6cb7c09b7e4/f/77b0aa523f7241fc92aadb2b01f762f4 for snapshot=snapshot_1694032547070_default_test-1694032518189 2023-09-06T20:35:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=45 2023-09-06T20:35:48,066 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1694032518189,,1694032518692.c5da9ae4b22d80dadf15a6cb7c09b7e4. 2023-09-06T20:35:48,066 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=46 2023-09-06T20:35:48,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.HMaster(3995): Remote procedure done, pid=46 2023-09-06T20:35:48,067 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1694032547070_default_test-1694032518189 on region c5da9ae4b22d80dadf15a6cb7c09b7e4 2023-09-06T20:35:48,069 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=46, ppid=45, state=RUNNABLE; SnapshotRegionProcedure c5da9ae4b22d80dadf15a6cb7c09b7e4 2023-09-06T20:35:48,071 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=46, resume processing ppid=45 2023-09-06T20:35:48,071 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=46, ppid=45, state=SUCCESS; SnapshotRegionProcedure c5da9ae4b22d80dadf15a6cb7c09b7e4 in 564 msec 2023-09-06T20:35:48,071 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1694032547070_default_test-1694032518189 table=test-1694032518189 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:35:48,072 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1694032547070_default_test-1694032518189 table=test-1694032518189 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:35:48,072 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1694032547070_default_test-1694032518189 table=test-1694032518189 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:35:48,072 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1694032547070_default_test-1694032518189 2023-09-06T20:35:48,073 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/.hbase-snapshot/.tmp/snapshot_1694032547070_default_test-1694032518189 2023-09-06T20:35:48,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=45 2023-09-06T20:35:48,483 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1694032547070_default_test-1694032518189 table=test-1694032518189 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:35:48,490 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1694032547070_default_test-1694032518189 table=test-1694032518189 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:35:48,490 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/.hbase-snapshot/.tmp/snapshot_1694032547070_default_test-1694032518189 to hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/.hbase-snapshot/snapshot_1694032547070_default_test-1694032518189 2023-09-06T20:35:48,491 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1694032547070_default_test-1694032518189 table=test-1694032518189 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:35:48,491 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1694032547070_default_test-1694032518189 table=test-1694032518189 type=FLUSH ttl=0 }, snapshot procedure id = 45 2023-09-06T20:35:48,492 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=45, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1694032547070_default_test-1694032518189 table=test-1694032518189 type=FLUSH ttl=0 } in 1.3990 sec 2023-09-06T20:35:49,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=45 2023-09-06T20:35:49,201 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: default:test-1694032518189, procId: 45 completed 2023-09-06T20:35:49,201 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1694032542698 2023-09-06T20:35:49,202 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-09-06T20:35:49,202 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-09-06T20:35:49,202 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1694032542698_test-1694032518189 2023-09-06T20:35:49,202 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1694032547070_default_test-1694032518189 to hdfs://localhost:41733/backupUT/backup_1694032542698/default/test-1694032518189/ 2023-09-06T20:35:49,202 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-09-06T20:35:49,226 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:36869, inputRoot=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd 2023-09-06T20:35:49,226 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,33223,1694032501568_-1684730672_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:41733/backupUT/backup_1694032542698/default/test-1694032518189, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:41733/backupUT/backup_1694032542698/default/test-1694032518189/.hbase-snapshot/.tmp/snapshot_1694032547070_default_test-1694032518189 2023-09-06T20:35:49,226 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:36869, inputRoot=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd, snapshotDir=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/.hbase-snapshot/snapshot_1694032547070_default_test-1694032518189. 2023-09-06T20:35:49,239 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/.hbase-snapshot/snapshot_1694032547070_default_test-1694032518189 to hdfs://localhost:41733/backupUT/backup_1694032542698/default/test-1694032518189/.hbase-snapshot/.tmp/snapshot_1694032547070_default_test-1694032518189 2023-09-06T20:35:49,665 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:49,666 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:49,666 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:49,667 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:49,681 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-09-06T20:35:49,681 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-09-06T20:35:50,675 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/hadoop-413656387594746601.jar 2023-09-06T20:35:50,675 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:50,676 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:50,676 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:50,677 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:50,677 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:50,677 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:50,678 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:50,678 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:50,679 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:50,679 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:35:50,680 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:35:50,680 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:35:50,681 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:35:50,682 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:35:50,682 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:35:50,682 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:35:50,683 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:35:50,683 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:35:50,684 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:35:50,684 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:35:50,684 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:35:50,685 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:35:50,686 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:35:50,686 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:35:50,687 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:35:50,687 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:35:50,687 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:35:50,688 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:35:50,715 WARN [Listener at jenkins-hbase3.apache.org/34023 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-09-06T20:35:50,739 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1694032547070_default_test-1694032518189' hfile list 2023-09-06T20:35:50,743 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-09-06T20:35:50,835 WARN [Listener at jenkins-hbase3.apache.org/34023 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/mapreduce.cluster.local.dir/1694032550789/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:35:50,836 WARN [Listener at jenkins-hbase3.apache.org/34023 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/mapreduce.cluster.local.dir/1694032550789/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:35:50,899 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-09-06T20:35:50,925 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/test-1694032518189/c5da9ae4b22d80dadf15a6cb7c09b7e4/f/77b0aa523f7241fc92aadb2b01f762f4 output=hdfs://localhost:41733/backupUT/backup_1694032542698/default/test-1694032518189/archive/data/default/test-1694032518189/c5da9ae4b22d80dadf15a6cb7c09b7e4/f/77b0aa523f7241fc92aadb2b01f762f4 2023-09-06T20:35:50,925 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 8.007M/sec 2023-09-06T20:35:51,877 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-09-06T20:35:51,878 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-09-06T20:35:51,886 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1694032547070_default_test-1694032518189 2023-09-06T20:35:51,886 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1694032547070_default_test-1694032518189 finished. 2023-09-06T20:35:51,886 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.BackupSystemTable(1014): test-1694032518189 2023-09-06T20:35:51,894 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1694032542698 2023-09-06T20:35:51,894 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-09-06T20:35:52,299 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:41733/backupUT/backup_1694032542698/.backup.manifest 2023-09-06T20:35:52,299 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.TableBackupClient(358): Backup backup_1694032542698 finished: type=FULL,tablelist=test-1694032518189,targetRootDir=hdfs://localhost:41733/backupUT,startts=1694032546846,completets=1694032551894,bytescopied=0 2023-09-06T20:35:52,300 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-09-06T20:35:52,300 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1694032547070_default_test-1694032518189 2023-09-06T20:35:52,301 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1694032547070_default_test-1694032518189" 2023-09-06T20:35:52,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1694032547070_default_test-1694032518189 2023-09-06T20:35:52,304 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1694032547070_default_test-1694032518189 for backup backup_1694032542698 succeeded. 2023-09-06T20:35:52,305 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-09-06T20:35:52,310 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-09-06T20:35:52,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-09-06T20:35:52,313 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-09-06T20:35:52,315 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-09-06T20:35:52,321 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] impl.TableBackupClient(376): Backup backup_1694032542698 completed. 2023-09-06T20:35:52,321 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-06T20:35:52,322 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3754a699 to 127.0.0.1:50423 2023-09-06T20:35:52,322 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:52,322 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] backup.TestRemoteRestore(94): backup complete 2023-09-06T20:35:52,333 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/hadoop.log.dir so I do NOT create it in target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6 2023-09-06T20:35:52,333 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/hadoop.tmp.dir so I do NOT create it in target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6 2023-09-06T20:35:52,333 WARN [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.HBaseTestingUtility(455): hadoop.tmp.dir property value differs in configuration and system: Configuration=/tmp/hadoop-jenkins while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/hadoop.tmp.dir Erasing configuration value by system value. 2023-09-06T20:35:52,333 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6 2023-09-06T20:35:52,333 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/cluster_b294908c-ea89-86ee-324a-909f7c935e98, deleteOnExit=true 2023-09-06T20:35:52,334 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/test.cache.data in system properties and HBase conf 2023-09-06T20:35:52,334 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/hadoop.tmp.dir in system properties and HBase conf 2023-09-06T20:35:52,334 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/hadoop.log.dir in system properties and HBase conf 2023-09-06T20:35:52,334 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/mapreduce.cluster.local.dir in system properties and HBase conf 2023-09-06T20:35:52,334 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-09-06T20:35:52,334 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-09-06T20:35:52,334 DEBUG [Listener at jenkins-hbase3.apache.org/34023 {}] fs.HFileSystem(308): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-09-06T20:35:52,334 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-09-06T20:35:52,335 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-09-06T20:35:52,335 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-09-06T20:35:52,335 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-09-06T20:35:52,335 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-09-06T20:35:52,335 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-09-06T20:35:52,335 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-09-06T20:35:52,335 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/dfs.journalnode.edits.dir in system properties and HBase conf 2023-09-06T20:35:52,335 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-09-06T20:35:52,335 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/nfs.dump.dir in system properties and HBase conf 2023-09-06T20:35:52,335 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/java.io.tmpdir in system properties and HBase conf 2023-09-06T20:35:52,336 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/dfs.journalnode.edits.dir in system properties and HBase conf 2023-09-06T20:35:52,336 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-09-06T20:35:52,336 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-09-06T20:35:52,348 WARN [Listener at jenkins-hbase3.apache.org/34023 {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-09-06T20:35:52,348 WARN [Listener at jenkins-hbase3.apache.org/34023 {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-09-06T20:35:52,397 WARN [Listener at jenkins-hbase3.apache.org/34023 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:35:52,399 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:35:52,405 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/java.io.tmpdir/Jetty_localhost_45079_hdfs____.ngfwit/webapp 2023-09-06T20:35:52,508 INFO [Listener at jenkins-hbase3.apache.org/34023 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:45079 2023-09-06T20:35:52,527 WARN [Listener at jenkins-hbase3.apache.org/34023 {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-09-06T20:35:52,527 WARN [Listener at jenkins-hbase3.apache.org/34023 {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-09-06T20:35:52,584 WARN [Listener at localhost/44107 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-09-06T20:35:52,600 WARN [Listener at localhost/44107 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-09-06T20:35:52,603 WARN [Listener at localhost/44107 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:35:52,604 INFO [Listener at localhost/44107 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:35:52,611 INFO [Listener at localhost/44107 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/java.io.tmpdir/Jetty_localhost_42999_datanode____uovs5e/webapp 2023-09-06T20:35:52,712 INFO [Listener at localhost/44107 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:42999 2023-09-06T20:35:52,730 WARN [Listener at localhost/46447 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-09-06T20:35:52,749 WARN [Listener at localhost/46447 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-09-06T20:35:52,751 WARN [Listener at localhost/46447 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:35:52,752 INFO [Listener at localhost/46447 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:35:52,755 INFO [Listener at localhost/46447 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/java.io.tmpdir/Jetty_localhost_36607_datanode____.fb0g4f/webapp 2023-09-06T20:35:52,815 WARN [Thread-1113 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-09-06T20:35:52,849 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x3cc27a0f0eafc77f: Processing first storage report for DS-aa84f34c-4deb-481d-accf-4f370ee2cf27 from datanode e907f9a8-d2b0-432f-a889-8a223f15ea7f 2023-09-06T20:35:52,849 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x3cc27a0f0eafc77f: from storage DS-aa84f34c-4deb-481d-accf-4f370ee2cf27 node DatanodeRegistration(127.0.0.1:46699, datanodeUuid=e907f9a8-d2b0-432f-a889-8a223f15ea7f, infoPort=33769, infoSecurePort=0, ipcPort=46447, storageInfo=lv=-57;cid=testClusterID;nsid=1028052491;c=1694032552350), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2023-09-06T20:35:52,849 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x3cc27a0f0eafc77f: Processing first storage report for DS-35228c74-3266-4a6f-a691-b54992c9034c from datanode e907f9a8-d2b0-432f-a889-8a223f15ea7f 2023-09-06T20:35:52,849 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x3cc27a0f0eafc77f: from storage DS-35228c74-3266-4a6f-a691-b54992c9034c node DatanodeRegistration(127.0.0.1:46699, datanodeUuid=e907f9a8-d2b0-432f-a889-8a223f15ea7f, infoPort=33769, infoSecurePort=0, ipcPort=46447, storageInfo=lv=-57;cid=testClusterID;nsid=1028052491;c=1694032552350), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-09-06T20:35:52,871 INFO [Listener at localhost/46447 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:36607 2023-09-06T20:35:52,888 WARN [Listener at localhost/38931 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-09-06T20:35:52,934 WARN [Listener at localhost/38931 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-09-06T20:35:52,936 WARN [Listener at localhost/38931 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:35:52,939 INFO [Listener at localhost/38931 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:35:52,945 INFO [Listener at localhost/38931 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/java.io.tmpdir/Jetty_localhost_43791_datanode____b7aaaf/webapp 2023-09-06T20:35:52,988 WARN [Thread-1148 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-09-06T20:35:53,014 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x4afce8330cf379c0: Processing first storage report for DS-44b37a56-4c62-4ba7-ba5c-1ae383d4b630 from datanode 3773ff95-89a1-444c-895b-8ae8dd8d459a 2023-09-06T20:35:53,015 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x4afce8330cf379c0: from storage DS-44b37a56-4c62-4ba7-ba5c-1ae383d4b630 node DatanodeRegistration(127.0.0.1:35183, datanodeUuid=3773ff95-89a1-444c-895b-8ae8dd8d459a, infoPort=36231, infoSecurePort=0, ipcPort=38931, storageInfo=lv=-57;cid=testClusterID;nsid=1028052491;c=1694032552350), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2023-09-06T20:35:53,015 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x4afce8330cf379c0: Processing first storage report for DS-0a71cdb7-d78a-44c8-80c8-4fdbe6ba0fb7 from datanode 3773ff95-89a1-444c-895b-8ae8dd8d459a 2023-09-06T20:35:53,015 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x4afce8330cf379c0: from storage DS-0a71cdb7-d78a-44c8-80c8-4fdbe6ba0fb7 node DatanodeRegistration(127.0.0.1:35183, datanodeUuid=3773ff95-89a1-444c-895b-8ae8dd8d459a, infoPort=36231, infoSecurePort=0, ipcPort=38931, storageInfo=lv=-57;cid=testClusterID;nsid=1028052491;c=1694032552350), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-09-06T20:35:53,053 INFO [Listener at localhost/38931 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:43791 2023-09-06T20:35:53,071 WARN [Listener at localhost/34359 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-09-06T20:35:53,150 WARN [Thread-1182 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-09-06T20:35:53,176 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xd2c13df22a50cae3: Processing first storage report for DS-41bb6fe0-6c27-443b-903c-d80a53e9aa19 from datanode 756345e9-486c-477e-b06e-982706522b8e 2023-09-06T20:35:53,176 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xd2c13df22a50cae3: from storage DS-41bb6fe0-6c27-443b-903c-d80a53e9aa19 node DatanodeRegistration(127.0.0.1:33773, datanodeUuid=756345e9-486c-477e-b06e-982706522b8e, infoPort=34219, infoSecurePort=0, ipcPort=34359, storageInfo=lv=-57;cid=testClusterID;nsid=1028052491;c=1694032552350), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2023-09-06T20:35:53,177 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xd2c13df22a50cae3: Processing first storage report for DS-e85d6c96-c0fe-4e4f-92b4-9692f8fa9fca from datanode 756345e9-486c-477e-b06e-982706522b8e 2023-09-06T20:35:53,177 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xd2c13df22a50cae3: from storage DS-e85d6c96-c0fe-4e4f-92b4-9692f8fa9fca node DatanodeRegistration(127.0.0.1:33773, datanodeUuid=756345e9-486c-477e-b06e-982706522b8e, infoPort=34219, infoSecurePort=0, ipcPort=34359, storageInfo=lv=-57;cid=testClusterID;nsid=1028052491;c=1694032552350), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-09-06T20:35:53,181 DEBUG [Listener at localhost/34359 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6 2023-09-06T20:35:53,181 INFO [Listener at localhost/34359 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-09-06T20:35:53,181 INFO [Listener at localhost/34359 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/test.cache.data in system properties and HBase conf 2023-09-06T20:35:53,181 INFO [Listener at localhost/34359 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/hadoop.tmp.dir in system properties and HBase conf 2023-09-06T20:35:53,181 INFO [Listener at localhost/34359 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/hadoop.log.dir in system properties and HBase conf 2023-09-06T20:35:53,181 INFO [Listener at localhost/34359 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/mapreduce.cluster.local.dir in system properties and HBase conf 2023-09-06T20:35:53,181 INFO [Listener at localhost/34359 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-09-06T20:35:53,181 INFO [Listener at localhost/34359 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-09-06T20:35:53,182 INFO [Listener at localhost/34359 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:35:53,184 INFO [Listener at localhost/34359 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:35:53,186 INFO [Listener at localhost/34359 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-09-06T20:35:53,186 INFO [Listener at localhost/34359 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-09-06T20:35:53,186 INFO [Listener at localhost/34359 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-09-06T20:35:53,186 INFO [Listener at localhost/34359 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-09-06T20:35:53,187 INFO [Listener at localhost/34359 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-09-06T20:35:53,187 INFO [Listener at localhost/34359 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-09-06T20:35:53,187 INFO [Listener at localhost/34359 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-09-06T20:35:53,187 INFO [Listener at localhost/34359 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/dfs.journalnode.edits.dir in system properties and HBase conf 2023-09-06T20:35:53,187 INFO [Listener at localhost/34359 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-09-06T20:35:53,187 INFO [Listener at localhost/34359 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/nfs.dump.dir in system properties and HBase conf 2023-09-06T20:35:53,187 INFO [Listener at localhost/34359 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/java.io.tmpdir in system properties and HBase conf 2023-09-06T20:35:53,187 INFO [Listener at localhost/34359 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/dfs.journalnode.edits.dir in system properties and HBase conf 2023-09-06T20:35:53,187 INFO [Listener at localhost/34359 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-09-06T20:35:53,187 INFO [Listener at localhost/34359 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-09-06T20:35:53,711 WARN [Thread-1301 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:35:53,716 WARN [Thread-1301 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-09-06T20:35:53,716 INFO [Thread-1301 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:35:53,725 INFO [Thread-1301 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_38895_jobhistory____.go4u1z/webapp 2023-09-06T20:35:53,781 INFO [Thread-1301 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-09-06T20:35:53,781 INFO [Thread-1301 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-09-06T20:35:53,781 INFO [Thread-1301 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-09-06T20:35:53,782 INFO [Thread-1301 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-09-06T20:35:53,789 INFO [Thread-1301 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:35:53,825 INFO [Thread-1301 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:35:53,876 INFO [Thread-1301 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-09-06T20:35:53,883 INFO [Thread-1301 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:38895 2023-09-06T20:35:54,704 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-09-06T20:35:54,704 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-09-06T20:35:55,201 WARN [Listener at jenkins-hbase3.apache.org/44501 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:35:55,210 WARN [Listener at jenkins-hbase3.apache.org/44501 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-09-06T20:35:55,211 INFO [Listener at jenkins-hbase3.apache.org/44501 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:35:55,222 INFO [Listener at jenkins-hbase3.apache.org/44501 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_35085_cluster____.aovcco/webapp 2023-09-06T20:35:55,275 INFO [Listener at jenkins-hbase3.apache.org/44501 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-09-06T20:35:55,276 INFO [Listener at jenkins-hbase3.apache.org/44501 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-09-06T20:35:55,276 INFO [Listener at jenkins-hbase3.apache.org/44501 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-09-06T20:35:55,276 INFO [Listener at jenkins-hbase3.apache.org/44501 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-09-06T20:35:55,283 INFO [Listener at jenkins-hbase3.apache.org/44501 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:35:55,421 INFO [Listener at jenkins-hbase3.apache.org/44501 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:35:55,494 INFO [Listener at jenkins-hbase3.apache.org/44501 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:35:55,501 INFO [Listener at jenkins-hbase3.apache.org/44501 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:35085 2023-09-06T20:35:55,702 WARN [Listener at jenkins-hbase3.apache.org/43735 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:35:55,708 WARN [Listener at jenkins-hbase3.apache.org/43735 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-09-06T20:35:55,708 INFO [Listener at jenkins-hbase3.apache.org/43735 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:35:55,715 INFO [Listener at jenkins-hbase3.apache.org/43735 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_38977_node____.50bdl9/webapp 2023-09-06T20:35:55,765 INFO [Listener at jenkins-hbase3.apache.org/43735 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-09-06T20:35:55,765 INFO [Listener at jenkins-hbase3.apache.org/43735 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-09-06T20:35:55,765 INFO [Listener at jenkins-hbase3.apache.org/43735 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-09-06T20:35:55,765 INFO [Listener at jenkins-hbase3.apache.org/43735 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-09-06T20:35:55,772 INFO [Listener at jenkins-hbase3.apache.org/43735 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:35:55,789 INFO [Listener at jenkins-hbase3.apache.org/43735 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:35:55,842 INFO [Listener at jenkins-hbase3.apache.org/43735 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:35:55,854 INFO [Listener at jenkins-hbase3.apache.org/43735 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:38977 2023-09-06T20:35:55,954 WARN [Listener at jenkins-hbase3.apache.org/46281 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:35:55,960 WARN [Listener at jenkins-hbase3.apache.org/46281 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-09-06T20:35:55,965 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:35:55,972 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_43607_node____kc6y2l/webapp 2023-09-06T20:35:56,029 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-09-06T20:35:56,029 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-09-06T20:35:56,030 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-09-06T20:35:56,030 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-09-06T20:35:56,036 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:35:56,051 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:35:56,105 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:35:56,112 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:43607 2023-09-06T20:35:56,117 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-09-06T20:35:56,118 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x660d8b73 to 127.0.0.1:50423 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:35:56,126 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ac70904, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:56,127 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:56,131 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:59846, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:56,133 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] impl.RestoreTablesClient(101): HBase table table1 does not exist. It will be created during restore process 2023-09-06T20:35:56,133 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41733/backupUT/backup_1694032542698/.backup.manifest 2023-09-06T20:35:56,136 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1694032542698/.backup.manifest 2023-09-06T20:35:56,137 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41733/backupUT/backup_1694032542698/.backup.manifest 2023-09-06T20:35:56,139 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1694032542698/.backup.manifest 2023-09-06T20:35:56,139 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] impl.RestoreTablesClient(148): Restoring 'test-1694032518189' to 'table1' from full backup image hdfs://localhost:41733/backupUT/backup_1694032542698/default/test-1694032518189 2023-09-06T20:35:56,148 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] util.RestoreTool(487): Creating target table 'table1' 2023-09-06T20:35:56,148 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:41733/backupUT/backup_1694032542698/default/test-1694032518189/archive/data/default/test-1694032518189/c5da9ae4b22d80dadf15a6cb7c09b7e4 2023-09-06T20:35:56,149 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:41733/backupUT/backup_1694032542698/default/test-1694032518189/archive/data/default/test-1694032518189/c5da9ae4b22d80dadf15a6cb7c09b7e4/f in region [hdfs://localhost:41733/backupUT/backup_1694032542698/default/test-1694032518189/archive/data/default/test-1694032518189/c5da9ae4b22d80dadf15a6cb7c09b7e4] 2023-09-06T20:35:56,153 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] util.RestoreTool(442): Trying to figure out region boundaries hfile=hdfs://localhost:41733/backupUT/backup_1694032542698/default/test-1694032518189/archive/data/default/test-1694032518189/c5da9ae4b22d80dadf15a6cb7c09b7e4/f/77b0aa523f7241fc92aadb2b01f762f4 first=row0 last=row98 2023-09-06T20:35:56,153 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:35:56,162 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:43240, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:35:56,163 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:35:56,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] procedure2.ProcedureExecutor(1032): Stored pid=47, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=table1 2023-09-06T20:35:56,169 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=47, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:35:56,169 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 47 2023-09-06T20:35:56,170 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=47, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:35:56,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-09-06T20:35:56,198 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => d6c0d26c67b01cb506cb4d2e384de381, NAME => 'table1,,1694032556163.d6c0d26c67b01cb506cb4d2e384de381.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd 2023-09-06T20:35:56,209 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(888): Instantiated table1,,1694032556163.d6c0d26c67b01cb506cb4d2e384de381.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:56,210 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1668): Closing d6c0d26c67b01cb506cb4d2e384de381, disabling compactions & flushes 2023-09-06T20:35:56,210 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1690): Closing region table1,,1694032556163.d6c0d26c67b01cb506cb4d2e384de381. 2023-09-06T20:35:56,210 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1694032556163.d6c0d26c67b01cb506cb4d2e384de381. 2023-09-06T20:35:56,210 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on table1,,1694032556163.d6c0d26c67b01cb506cb4d2e384de381. after waiting 0 ms 2023-09-06T20:35:56,210 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region table1,,1694032556163.d6c0d26c67b01cb506cb4d2e384de381. 2023-09-06T20:35:56,210 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1909): Closed table1,,1694032556163.d6c0d26c67b01cb506cb4d2e384de381. 2023-09-06T20:35:56,210 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1622): Region close journal for d6c0d26c67b01cb506cb4d2e384de381: 2023-09-06T20:35:56,211 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=47, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:35:56,211 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"table1,,1694032556163.d6c0d26c67b01cb506cb4d2e384de381.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1694032556211"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032556211"}]},"ts":"1694032556211"} 2023-09-06T20:35:56,213 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:35:56,214 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=47, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:35:56,214 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032556214"}]},"ts":"1694032556214"} 2023-09-06T20:35:56,216 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLING in hbase:meta 2023-09-06T20:35:56,220 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=d6c0d26c67b01cb506cb4d2e384de381, ASSIGN}] 2023-09-06T20:35:56,223 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=d6c0d26c67b01cb506cb4d2e384de381, ASSIGN 2023-09-06T20:35:56,224 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=table1, region=d6c0d26c67b01cb506cb4d2e384de381, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36075,1694032504561; forceNewPlan=false, retain=false 2023-09-06T20:35:56,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-09-06T20:35:56,377 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=48 updating hbase:meta row=d6c0d26c67b01cb506cb4d2e384de381, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:56,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE; OpenRegionProcedure d6c0d26c67b01cb506cb4d2e384de381, server=jenkins-hbase3.apache.org,36075,1694032504561}] 2023-09-06T20:35:56,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-09-06T20:35:56,534 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(130): Open table1,,1694032556163.d6c0d26c67b01cb506cb4d2e384de381. 2023-09-06T20:35:56,534 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7254): Opening region: {ENCODED => d6c0d26c67b01cb506cb4d2e384de381, NAME => 'table1,,1694032556163.d6c0d26c67b01cb506cb4d2e384de381.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:35:56,535 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:35:56,535 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 d6c0d26c67b01cb506cb4d2e384de381 2023-09-06T20:35:56,535 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(888): Instantiated table1,,1694032556163.d6c0d26c67b01cb506cb4d2e384de381.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:56,535 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7296): checking encryption for d6c0d26c67b01cb506cb4d2e384de381 2023-09-06T20:35:56,535 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7299): checking classloading for d6c0d26c67b01cb506cb4d2e384de381 2023-09-06T20:35:56,536 INFO [StoreOpener-d6c0d26c67b01cb506cb4d2e384de381-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region d6c0d26c67b01cb506cb4d2e384de381 2023-09-06T20:35:56,538 INFO [StoreOpener-d6c0d26c67b01cb506cb4d2e384de381-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d6c0d26c67b01cb506cb4d2e384de381 columnFamilyName f 2023-09-06T20:35:56,538 DEBUG [StoreOpener-d6c0d26c67b01cb506cb4d2e384de381-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:56,538 INFO [StoreOpener-d6c0d26c67b01cb506cb4d2e384de381-1 {}] regionserver.HStore(324): Store=d6c0d26c67b01cb506cb4d2e384de381/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:35:56,539 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/table1/d6c0d26c67b01cb506cb4d2e384de381 2023-09-06T20:35:56,540 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/table1/d6c0d26c67b01cb506cb4d2e384de381 2023-09-06T20:35:56,542 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1076): writing seq id for d6c0d26c67b01cb506cb4d2e384de381 2023-09-06T20:35:56,544 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/table1/d6c0d26c67b01cb506cb4d2e384de381/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:35:56,545 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): Opened d6c0d26c67b01cb506cb4d2e384de381; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=12051402560, jitterRate=0.12237432599067688}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:35:56,546 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(992): Region open journal for d6c0d26c67b01cb506cb4d2e384de381: 2023-09-06T20:35:56,546 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2556): Post open deploy tasks for table1,,1694032556163.d6c0d26c67b01cb506cb4d2e384de381., pid=49, masterSystemTime=1694032556531 2023-09-06T20:35:56,548 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2583): Finished post open deploy task for table1,,1694032556163.d6c0d26c67b01cb506cb4d2e384de381. 2023-09-06T20:35:56,548 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(158): Opened table1,,1694032556163.d6c0d26c67b01cb506cb4d2e384de381. 2023-09-06T20:35:56,548 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=48 updating hbase:meta row=d6c0d26c67b01cb506cb4d2e384de381, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:35:56,551 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=49, resume processing ppid=48 2023-09-06T20:35:56,551 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=49, ppid=48, state=SUCCESS; OpenRegionProcedure d6c0d26c67b01cb506cb4d2e384de381, server=jenkins-hbase3.apache.org,36075,1694032504561 in 171 msec 2023-09-06T20:35:56,553 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=48, resume processing ppid=47 2023-09-06T20:35:56,553 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=48, ppid=47, state=SUCCESS; TransitRegionStateProcedure table=table1, region=d6c0d26c67b01cb506cb4d2e384de381, ASSIGN in 331 msec 2023-09-06T20:35:56,553 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=47, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:35:56,553 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032556553"}]},"ts":"1694032556553"} 2023-09-06T20:35:56,554 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLED in hbase:meta 2023-09-06T20:35:56,557 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=47, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:35:56,558 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=47, state=SUCCESS; CreateTableProcedure table=table1 in 394 msec 2023-09-06T20:35:56,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-09-06T20:35:56,774 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:table1, procId: 47 completed 2023-09-06T20:35:56,783 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:41733/backupUT/backup_1694032542698/default/test-1694032518189/archive/data/default/test-1694032518189/c5da9ae4b22d80dadf15a6cb7c09b7e4 from hbase tables test-1694032518189 to tables table1 2023-09-06T20:35:56,783 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.MapReduceRestoreJob(72): Restore test-1694032518189 into table1 2023-09-06T20:35:56,785 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:36869/backupUT/bulk_output-default-table1-1694032556783 from hdfs://localhost:41733/backupUT/backup_1694032542698/default/test-1694032518189/archive/data/default/test-1694032518189/c5da9ae4b22d80dadf15a6cb7c09b7e4 2023-09-06T20:35:56,786 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0c7848e6 to 127.0.0.1:50423 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:35:56,793 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@234614ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:56,793 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:35:56,795 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:43254, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:35:56,796 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-09-06T20:35:56,796 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table table1 2023-09-06T20:35:56,801 DEBUG [hconnection-0x36d14f3d-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:56,803 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:59858, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:56,805 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for table1: 2023-09-06T20:35:56,805 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-09-06T20:35:56,805 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to /user/jenkins/hbase-staging/partitions_fe7f637a-ae18-41a8-9abc-df6a2feb803d 2023-09-06T20:35:56,815 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:56,816 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:56,816 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:56,817 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:57,795 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/hadoop-7268136503632233792.jar 2023-09-06T20:35:57,795 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:57,796 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:57,796 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:57,797 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:57,797 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:57,797 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:57,798 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:57,798 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:57,799 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:57,799 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:35:57,800 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:35:57,800 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:35:57,801 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:35:57,801 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:35:57,804 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:35:57,804 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:35:57,804 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:35:57,805 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:35:57,805 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:35:57,806 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:35:57,806 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:35:57,807 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:57,807 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:57,808 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:57,808 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:57,808 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:57,808 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:57,809 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:35:57,809 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: table1 2023-09-06T20:35:57,809 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-06T20:35:57,810 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0c7848e6 to 127.0.0.1:50423 2023-09-06T20:35:57,810 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:57,810 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-09-06T20:35:57,810 WARN [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-09-06T20:35:57,811 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:35:57,998 WARN [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-09-06T20:35:58,423 WARN [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-09-06T20:35:58,761 WARN [SchedulerEventDispatcher:Event Processor {}] capacity.LeafQueue(815): maximum-am-resource-percent is insufficient to start a single application in queue, it is likely set too low. skipping enforcement to allow at least one application to start 2023-09-06T20:35:58,762 WARN [SchedulerEventDispatcher:Event Processor {}] capacity.LeafQueue(851): maximum-am-resource-percent is insufficient to start a single application in queue for user, it is likely set too low. skipping enforcement to allow at least one application to start 2023-09-06T20:35:59,040 INFO [Socket Reader #1 for port 0 {}] ipc.Server$Connection(2001): Auth successful for appattempt_1694032555510_0001_000001 (auth:SIMPLE) 2023-09-06T20:36:01,087 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'table1' 2023-09-06T20:36:02,347 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:36:02,856 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2023-09-06T20:36:02,856 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_table1 Metrics about Tables on a single HBase RegionServer 2023-09-06T20:36:04,175 INFO [Socket Reader #1 for port 0 {}] ipc.Server$Connection(2001): Auth successful for appattempt_1694032555510_0001_000001 (auth:SIMPLE) 2023-09-06T20:36:06,446 INFO [Socket Reader #1 for port 0 {}] ipc.Server$Connection(2001): Auth successful for appattempt_1694032555510_0001_000001 (auth:SIMPLE) 2023-09-06T20:36:07,716 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region cd7d22af9ee99f89485a5c716f51a037 changed from -1.0 to 0.0, refreshing cache 2023-09-06T20:36:12,346 INFO [Socket Reader #1 for port 0 {}] ipc.Server$Connection(2001): Auth successful for appattempt_1694032555510_0001_000001 (auth:SIMPLE) 2023-09-06T20:36:13,571 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2023-09-06T20:36:13,571 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 932b31f1be4976500c94245ad49c9242 changed from -1.0 to 0.0, refreshing cache 2023-09-06T20:36:15,479 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49196, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:36:15,967 INFO [Socket Reader #1 for port 0 {}] ipc.Server$Connection(2001): Auth successful for appattempt_1694032555510_0001_000001 (auth:SIMPLE) 2023-09-06T20:36:15,985 WARN [ContainersLauncher #1 {}] nodemanager.DefaultContainerExecutor(305): Exit code from container container_1694032555510_0001_01_000003 is : 143 2023-09-06T20:36:17,163 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:36869/backupUT/bulk_output-default-table1-1694032556783 2023-09-06T20:36:17,164 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x75f9ac61 to 127.0.0.1:50423 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:36:17,172 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fedd88a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:36:17,173 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:36:17,176 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49208, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:36:17,184 WARN [Listener at jenkins-hbase3.apache.org/46281 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:36869/backupUT/bulk_output-default-table1-1694032556783/_SUCCESS 2023-09-06T20:36:17,189 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:36:17,192 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:53688, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:36:17,212 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:36869/backupUT/bulk_output-default-table1-1694032556783/f/0be56a1ab92b46f3879e1c866e5d727c first=Optional[row0] last=Optional[row98] 2023-09-06T20:36:17,214 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=table1,,1694032556163.d6c0d26c67b01cb506cb4d2e384de381., hostname=jenkins-hbase3.apache.org,36075,1694032504561, seqNum=2 for row with hfile group [{f,hdfs://localhost:36869/backupUT/bulk_output-default-table1-1694032556783/f/0be56a1ab92b46f3879e1c866e5d727c}] 2023-09-06T20:36:17,218 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:36869/backupUT/bulk_output-default-table1-1694032556783/f/0be56a1ab92b46f3879e1c866e5d727c for inclusion in d6c0d26c67b01cb506cb4d2e384de381/f 2023-09-06T20:36:17,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-09-06T20:36:17,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] regionserver.HStore(625): Region bounds: first= last= 2023-09-06T20:36:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] regionserver.HRegion(2520): Flush status journal for d6c0d26c67b01cb506cb4d2e384de381: 2023-09-06T20:36:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:36869/backupUT/bulk_output-default-table1-1694032556783/f/0be56a1ab92b46f3879e1c866e5d727c to hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/staging/jenkins__table1__n5ndj2ec00f744d6i5chfu1g4qbdd244p9052fr0tkf7l7ok1vfah81d10t4jn3n/f/0be56a1ab92b46f3879e1c866e5d727c 2023-09-06T20:36:17,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/staging/jenkins__table1__n5ndj2ec00f744d6i5chfu1g4qbdd244p9052fr0tkf7l7ok1vfah81d10t4jn3n/f/0be56a1ab92b46f3879e1c866e5d727c as hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/table1/d6c0d26c67b01cb506cb4d2e384de381/f/f1afbc1cbcef4d488474508e2e58b926_SeqId_4_ 2023-09-06T20:36:17,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x33786f6a to 127.0.0.1:50423 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:36:17,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bf5eafc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:36:17,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:36:17,235 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:53702, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:36:17,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:36:17,240 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49220, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:36:17,253 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-06T20:36:17,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x33786f6a to 127.0.0.1:50423 2023-09-06T20:36:17,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:36:17,254 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/staging/jenkins__table1__n5ndj2ec00f744d6i5chfu1g4qbdd244p9052fr0tkf7l7ok1vfah81d10t4jn3n/f/0be56a1ab92b46f3879e1c866e5d727c into d6c0d26c67b01cb506cb4d2e384de381/f as hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/table1/d6c0d26c67b01cb506cb4d2e384de381/f/f1afbc1cbcef4d488474508e2e58b926_SeqId_4_ - updating store file list. 2023-09-06T20:36:17,259 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/table1/d6c0d26c67b01cb506cb4d2e384de381/f/f1afbc1cbcef4d488474508e2e58b926_SeqId_4_ into d6c0d26c67b01cb506cb4d2e384de381/f 2023-09-06T20:36:17,259 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/staging/jenkins__table1__n5ndj2ec00f744d6i5chfu1g4qbdd244p9052fr0tkf7l7ok1vfah81d10t4jn3n/f/0be56a1ab92b46f3879e1c866e5d727c into d6c0d26c67b01cb506cb4d2e384de381/f (new location: hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/table1/d6c0d26c67b01cb506cb4d2e384de381/f/f1afbc1cbcef4d488474508e2e58b926_SeqId_4_) 2023-09-06T20:36:17,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/staging/jenkins__table1__n5ndj2ec00f744d6i5chfu1g4qbdd244p9052fr0tkf7l7ok1vfah81d10t4jn3n/f/0be56a1ab92b46f3879e1c866e5d727c 2023-09-06T20:36:17,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7e950a42 to 127.0.0.1:50423 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:36:17,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d4cf96a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:36:17,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:36:17,276 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:53714, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-09-06T20:36:17,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:36:17,282 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49230, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-09-06T20:36:17,299 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-06T20:36:17,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7e950a42 to 127.0.0.1:50423 2023-09-06T20:36:17,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36075 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:36:17,301 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-06T20:36:17,301 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x75f9ac61 to 127.0.0.1:50423 2023-09-06T20:36:17,301 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:36:17,301 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-09-06T20:36:17,302 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-09-06T20:36:17,302 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] impl.RestoreTablesClient(233): Backup: backup_1694032542698 hdfs://localhost:41733/backupUT/backup_1694032542698/default/test-1694032518189/ 2023-09-06T20:36:17,302 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-09-06T20:36:17,306 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] client.HBaseAdmin$18(967): Started disable of table1 2023-09-06T20:36:17,307 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable table1 2023-09-06T20:36:17,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] procedure2.ProcedureExecutor(1032): Stored pid=50, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=table1 2023-09-06T20:36:17,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-09-06T20:36:17,311 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032577311"}]},"ts":"1694032577311"} 2023-09-06T20:36:17,312 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLING in hbase:meta 2023-09-06T20:36:17,314 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set table1 to state=DISABLING 2023-09-06T20:36:17,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=d6c0d26c67b01cb506cb4d2e384de381, UNASSIGN}] 2023-09-06T20:36:17,316 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=51, ppid=50, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=d6c0d26c67b01cb506cb4d2e384de381, UNASSIGN 2023-09-06T20:36:17,317 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=51 updating hbase:meta row=d6c0d26c67b01cb506cb4d2e384de381, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:36:17,318 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE; CloseRegionProcedure d6c0d26c67b01cb506cb4d2e384de381, server=jenkins-hbase3.apache.org,36075,1694032504561}] 2023-09-06T20:36:17,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-09-06T20:36:17,470 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] handler.UnassignRegionHandler(119): Close d6c0d26c67b01cb506cb4d2e384de381 2023-09-06T20:36:17,470 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1668): Closing d6c0d26c67b01cb506cb4d2e384de381, disabling compactions & flushes 2023-09-06T20:36:17,470 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1690): Closing region table1,,1694032556163.d6c0d26c67b01cb506cb4d2e384de381. 2023-09-06T20:36:17,470 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1694032556163.d6c0d26c67b01cb506cb4d2e384de381. 2023-09-06T20:36:17,470 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1778): Acquired close lock on table1,,1694032556163.d6c0d26c67b01cb506cb4d2e384de381. after waiting 0 ms 2023-09-06T20:36:17,470 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1788): Updates disabled for region table1,,1694032556163.d6c0d26c67b01cb506cb4d2e384de381. 2023-09-06T20:36:17,478 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/table1/d6c0d26c67b01cb506cb4d2e384de381/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-09-06T20:36:17,479 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:36:17,480 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1909): Closed table1,,1694032556163.d6c0d26c67b01cb506cb4d2e384de381. 2023-09-06T20:36:17,480 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1622): Region close journal for d6c0d26c67b01cb506cb4d2e384de381: 2023-09-06T20:36:17,481 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] handler.UnassignRegionHandler(163): Closed d6c0d26c67b01cb506cb4d2e384de381 2023-09-06T20:36:17,482 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=51 updating hbase:meta row=d6c0d26c67b01cb506cb4d2e384de381, regionState=CLOSED 2023-09-06T20:36:17,485 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=52, resume processing ppid=51 2023-09-06T20:36:17,485 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=52, ppid=51, state=SUCCESS; CloseRegionProcedure d6c0d26c67b01cb506cb4d2e384de381, server=jenkins-hbase3.apache.org,36075,1694032504561 in 165 msec 2023-09-06T20:36:17,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=51, resume processing ppid=50 2023-09-06T20:36:17,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=51, ppid=50, state=SUCCESS; TransitRegionStateProcedure table=table1, region=d6c0d26c67b01cb506cb4d2e384de381, UNASSIGN in 171 msec 2023-09-06T20:36:17,487 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032577487"}]},"ts":"1694032577487"} 2023-09-06T20:36:17,488 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLED in hbase:meta 2023-09-06T20:36:17,490 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set table1 to state=DISABLED 2023-09-06T20:36:17,492 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=50, state=SUCCESS; DisableTableProcedure table=table1 in 184 msec 2023-09-06T20:36:17,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-09-06T20:36:17,613 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] client.HBaseAdmin$TableFuture(3677): Operation: DISABLE, Table Name: default:table1, procId: 50 completed 2023-09-06T20:36:17,614 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.HMaster$5(2448): Client=jenkins//172.31.12.81 delete table1 2023-09-06T20:36:17,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] procedure2.ProcedureExecutor(1032): Stored pid=53, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=table1 2023-09-06T20:36:17,618 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=53, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-09-06T20:36:17,619 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=53, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=table1 2023-09-06T20:36:17,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=53 2023-09-06T20:36:17,621 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/table1/d6c0d26c67b01cb506cb4d2e384de381 2023-09-06T20:36:17,623 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/table1/d6c0d26c67b01cb506cb4d2e384de381/f, FileablePath, hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/table1/d6c0d26c67b01cb506cb4d2e384de381/recovered.edits] 2023-09-06T20:36:17,627 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/table1/d6c0d26c67b01cb506cb4d2e384de381/f/f1afbc1cbcef4d488474508e2e58b926_SeqId_4_ to hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/archive/data/default/table1/d6c0d26c67b01cb506cb4d2e384de381/f/f1afbc1cbcef4d488474508e2e58b926_SeqId_4_ 2023-09-06T20:36:17,630 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/table1/d6c0d26c67b01cb506cb4d2e384de381/recovered.edits/6.seqid to hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/archive/data/default/table1/d6c0d26c67b01cb506cb4d2e384de381/recovered.edits/6.seqid 2023-09-06T20:36:17,631 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/table1/d6c0d26c67b01cb506cb4d2e384de381 2023-09-06T20:36:17,631 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(293): Archived table1 regions 2023-09-06T20:36:17,633 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=53, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=table1 2023-09-06T20:36:17,636 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of table1 from hbase:meta 2023-09-06T20:36:17,637 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(387): Removing 'table1' descriptor. 2023-09-06T20:36:17,638 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=53, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=table1 2023-09-06T20:36:17,638 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(377): Removing 'table1' from region states. 2023-09-06T20:36:17,639 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1,,1694032556163.d6c0d26c67b01cb506cb4d2e384de381.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1694032577638"}]},"ts":"9223372036854775807"} 2023-09-06T20:36:17,640 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-06T20:36:17,640 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => d6c0d26c67b01cb506cb4d2e384de381, NAME => 'table1,,1694032556163.d6c0d26c67b01cb506cb4d2e384de381.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:36:17,640 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(381): Marking 'table1' as deleted. 2023-09-06T20:36:17,640 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1694032577640"}]},"ts":"9223372036854775807"} 2023-09-06T20:36:17,642 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1678): Deleted table table1 state from META 2023-09-06T20:36:17,644 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(127): Finished pid=53, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-09-06T20:36:17,644 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=53, state=SUCCESS; DeleteTableProcedure table=table1 in 30 msec 2023-09-06T20:36:17,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33223 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=53 2023-09-06T20:36:17,720 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] client.HBaseAdmin$TableFuture(3677): Operation: DELETE, Table Name: default:table1, procId: 53 completed 2023-09-06T20:36:17,777 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] hbase.ResourceChecker(175): after: backup.TestRemoteRestore#testFullRestoreRemoteWithAlternateRestoreOutputDir Thread=1306 (was 830) Potentially hanging thread: IPC Server handler 2 on default port 36903 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: hconnection-0x1ea85617-shared-pool-16 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 14 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: RS-EventLoopGroup-5-3 org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:209) org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:202) org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:306) org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:363) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 29 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 374206074@qtp-2001179143-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: IPC Server handler 41 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 18 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: CacheReplicationMonitor(1871314891) sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) org.apache.hadoop.hdfs.server.blockmanagement.CacheReplicationMonitor.run(CacheReplicationMonitor.java:181) Potentially hanging thread: pool-350-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 15 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 36903 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: DeletionService #2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #48 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@7428b79f java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 3 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 36 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 28 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.hdfs.server.datanode.DataXceiverServer@4af66dff sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:421) sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:249) sun.nio.ch.ServerSocketAdaptor.accept(ServerSocketAdaptor.java:113) org.apache.hadoop.hdfs.net.TcpPeerServer.accept(TcpPeerServer.java:85) org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.run(DataXceiverServer.java:145) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-57 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: New I/O worker #49 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: pool-372-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-43 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 31 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 3 on default port 36903 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 37405 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 35 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50423@0x660d8b73 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:328) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$490/1081298586.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 45 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 47 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #40 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: pool-370-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/cluster_b294908c-ea89-86ee-324a-909f7c935e98/dfs/data/data1/current/BP-75859831-172.31.12.81-1694032552350 java.lang.Thread.sleep(Native Method) org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:205) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DeletionService #3 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-67 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 13 on default port 36215 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 20 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 150321230@qtp-1220827335-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:36607 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: IPC Server handler 10 on default port 36903 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: region-location-3 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 0 on default port 40825 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 22 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 26 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 45775 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 29 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/cluster_b294908c-ea89-86ee-324a-909f7c935e98/dfs/data/data5) java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:627) Potentially hanging thread: IPC Server handler 39 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-55 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 1 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-91703826_16 at /127.0.0.1:50138 [Receiving block BP-657318226-172.31.12.81-1694032498405:blk_1073741871_1047] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 26 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 36 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 4 on default port 46281 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 18 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 49 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 48 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: PacketResponder: BP-657318226-172.31.12.81-1694032498405:blk_1073741871_1047, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 0 on default port 44501 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: ContainersLauncher #1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 34 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 30 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 1 on default port 36215 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 44107 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@59d79676 java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: 2054540103@qtp-1932060979-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:45079 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: region-location-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-51 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 45 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 22 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: DeletionService #1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 25 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 19 on default port 36903 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 829208612@qtp-1687852186-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_116006268_16 at /127.0.0.1:42594 [Waiting for operation #8] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 43 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@298c6338 java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: 437121911@qtp-62646430-1 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: IPC Server handler 5 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 5 on default port 45775 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/cluster_b294908c-ea89-86ee-324a-909f7c935e98/dfs/data/data2) java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:627) Potentially hanging thread: IPC Server handler 9 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: ContainersLauncher #0 java.io.FileInputStream.readBytes(Native Method) java.io.FileInputStream.read(FileInputStream.java:255) java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.io.BufferedInputStream.read(BufferedInputStream.java:345) sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:284) sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:326) sun.nio.cs.StreamDecoder.read(StreamDecoder.java:178) java.io.InputStreamReader.read(InputStreamReader.java:184) java.io.BufferedReader.fill(BufferedReader.java:161) java.io.BufferedReader.read1(BufferedReader.java:212) java.io.BufferedReader.read(BufferedReader.java:286) org.apache.hadoop.util.Shell$ShellCommandExecutor.parseExecResult(Shell.java:1229) org.apache.hadoop.util.Shell.runCommand(Shell.java:984) org.apache.hadoop.util.Shell.run(Shell.java:884) org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1216) org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor.launchContainer(DefaultContainerExecutor.java:294) org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.launchContainer(ContainerLaunch.java:447) org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.call(ContainerLaunch.java:298) org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.call(ContainerLaunch.java:99) java.util.concurrent.FutureTask.run(FutureTask.java:266) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 11 on default port 36215 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 12 on default port 36903 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 19 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 30 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-39 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: Timer-50 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Client (366321005) connection to localhost/127.0.0.1:44107 from jenkins java.lang.Object.wait(Native Method) org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1053) org.apache.hadoop.ipc.Client$Connection.run(Client.java:1097) Potentially hanging thread: IPC Server handler 3 on default port 43735 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-657318226-172.31.12.81-1694032498405:blk_1073741870_1046, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 4 on default port 34359 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 13 on default port 36903 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 49 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 1 on default port 44107 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-63 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: New I/O worker #44 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 27 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #52 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 13 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Thread-1344 java.lang.Thread.sleep(Native Method) org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:143) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 25 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 44 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 42 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@6357e82e java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@4c2499b4 java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 8 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Thread-2051 java.io.FileInputStream.readBytes(Native Method) java.io.FileInputStream.read(FileInputStream.java:255) java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.io.BufferedInputStream.read(BufferedInputStream.java:345) sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:284) sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:326) sun.nio.cs.StreamDecoder.read(StreamDecoder.java:178) java.io.InputStreamReader.read(InputStreamReader.java:184) java.io.BufferedReader.fill(BufferedReader.java:161) java.io.BufferedReader.readLine(BufferedReader.java:324) java.io.BufferedReader.readLine(BufferedReader.java:389) org.apache.hadoop.util.Shell$1.run(Shell.java:955) Potentially hanging thread: IPC Server handler 10 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-42 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 4 on default port 36903 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: nioEventLoopGroup-10-1 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:803) io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457) io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-59 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 4 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 33 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 42 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 43 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 19 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-355-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-44 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: pool-353-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 39 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: hconnection-0x1ea85617-shared-pool-13 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 18 on default port 36903 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 4 on default port 43735 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 40699 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 0 on default port 38931 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 46447 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Thread[Thread-1337,5,FailOnTimeoutGroup] java.lang.Thread.sleep(Native Method) org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:694) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 1 on default port 46447 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 46281 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 35 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #62 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server idle connection scanner for port 40825 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: New I/O worker #55 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 23 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 5 on default port 36903 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 44107 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 40 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: HFileArchiver-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/cluster_b294908c-ea89-86ee-324a-909f7c935e98/dfs/data/data2/current/BP-75859831-172.31.12.81-1694032552350 java.lang.Thread.sleep(Native Method) org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:205) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/cluster_b294908c-ea89-86ee-324a-909f7c935e98/dfs/data/data4) java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:627) Potentially hanging thread: IPC Server handler 46 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 3 on default port 46281 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #46 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1102555593_1 at /127.0.0.1:40216 [Waiting for operation #2] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #37 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #61 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@7fcd46a7 java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 25 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-66 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 21 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 3 on default port 44107 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/cluster_b294908c-ea89-86ee-324a-909f7c935e98/dfs/data/data3/current/BP-75859831-172.31.12.81-1694032552350 java.lang.Thread.sleep(Native Method) org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:205) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-62 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: New I/O worker #63 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 42 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: LeaseRenewer:jenkins@localhost:44107 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:412) org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$600(LeaseRenewer.java:76) org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:308) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 12 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 38 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 21 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #67 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 1 on default port 36903 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 20 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 7 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 34 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-360-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 28 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 4 on default port 46447 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #35 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@17690a5[State = -1, empty queue] sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: BP-75859831-172.31.12.81-1694032552350 heartbeating to localhost/127.0.0.1:44107 java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.IncrementalBlockReportManager.waitTillNextIBR(IncrementalBlockReportManager.java:158) org.apache.hadoop.hdfs.server.datanode.BPServiceActor.offerService(BPServiceActor.java:715) org.apache.hadoop.hdfs.server.datanode.BPServiceActor.run(BPServiceActor.java:851) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #39 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 48 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 16 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #54 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 10 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 5 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 6 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #65 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 8 on default port 36215 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 49 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #66 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 3 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: ForkJoinPool-2-worker-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1824) java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1693) java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) Potentially hanging thread: IPC Server handler 46 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/cluster_b294908c-ea89-86ee-324a-909f7c935e98/dfs/data/data6/current/BP-75859831-172.31.12.81-1694032552350 java.lang.Thread.sleep(Native Method) org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:205) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 18 on default port 36215 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 3 on default port 34359 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 15 on default port 36903 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: hconnection-0x1ea85617-shared-pool-18 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: 398234696@qtp-1687852186-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:38895 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: IPC Server handler 14 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.hdfs.server.blockmanagement.PendingReplicationBlocks$PendingReplicationMonitor@43be33b2 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.server.blockmanagement.PendingReplicationBlocks$PendingReplicationMonitor.run(PendingReplicationBlocks.java:244) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 4 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 6 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #38 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@156bcbfe[State = -1, empty queue] sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 14 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-61 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 23 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 7 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Client (366321005) connection to localhost/127.0.0.1:44107 from jenkins java.lang.Object.wait(Native Method) org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1053) org.apache.hadoop.ipc.Client$Connection.run(Client.java:1097) Potentially hanging thread: IPC Server handler 1 on default port 38931 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 4 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 1 on default port 45775 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 17 on default port 36215 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 43735 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 16 on default port 36215 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 16 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 37 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 16 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 40 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 10 on default port 36215 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 45775 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 45775 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: BP-75859831-172.31.12.81-1694032552350 heartbeating to localhost/127.0.0.1:44107 java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.IncrementalBlockReportManager.waitTillNextIBR(IncrementalBlockReportManager.java:158) org.apache.hadoop.hdfs.server.datanode.BPServiceActor.offerService(BPServiceActor.java:715) org.apache.hadoop.hdfs.server.datanode.BPServiceActor.run(BPServiceActor.java:851) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 9 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 44501 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 17 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/cluster_b294908c-ea89-86ee-324a-909f7c935e98/dfs/data/data3) java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:627) Potentially hanging thread: IPC Server handler 14 on default port 36215 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #50 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 7 on default port 45775 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 28 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 44 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-64 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: nioEventLoopGroup-12-1 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:803) io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457) io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: pool-404-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 38 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 36215 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-45 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: hconnection-0x1ea85617-shared-pool-14 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 3 on default port 46447 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 38931 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: pool-400-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 7 on default port 36215 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 36941 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/cluster_b294908c-ea89-86ee-324a-909f7c935e98/dfs/data/data5/current/BP-75859831-172.31.12.81-1694032552350 java.lang.Thread.sleep(Native Method) org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:205) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: 1950359699@qtp-259296187-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:38977 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: IPC Server handler 13 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #59 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: 2038752608@qtp-1061392326-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: IPC Server handler 20 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #53 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 8 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O server boss #68 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.jboss.netty.channel.socket.nio.NioServerBoss.select(NioServerBoss.java:163) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.NioServerBoss.run(NioServerBoss.java:42) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/cluster_b294908c-ea89-86ee-324a-909f7c935e98/dfs/data/data1) java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:627) Potentially hanging thread: IPC Server handler 7 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 34 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 12 on default port 36215 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #57 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: 1957542328@qtp-1220827335-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: IPC Server handler 19 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 1 on default port 43735 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 38931 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 9 on default port 36903 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 43735 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 14 on default port 36903 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 11 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 27 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor@7425fa87 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor.run(LeaseManager.java:533) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 6 on default port 45775 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 13 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 3 on default port 36215 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 74385676@qtp-1932060979-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@3817eb29[State = -1, empty queue] sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeEditLogRoller@3bd3ab21 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeEditLogRoller.run(FSNamesystem.java:3883) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: pool-348-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: 1850045240@qtp-1061392326-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:35085 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: Timer-65 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: Timer-47 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 43 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeResourceMonitor@74ffff29 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeResourceMonitor.run(FSNamesystem.java:3841) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-41 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 2 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 33 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 1 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 43735 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 41 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-58 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 32 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #58 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server idle connection scanner for port 34359 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 9 on default port 36215 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 36903 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 24 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 33 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-54 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: New I/O worker #45 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: pool-402-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: 1847984170@qtp-2001179143-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:43607 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: IPC Server handler 8 on default port 45775 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Thread[Thread-1302,5,FailOnTimeoutGroup] java.lang.Thread.sleep(Native Method) org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:694) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server idle connection scanner for port 46281 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: Timer-49 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 24 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O server boss #51 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.jboss.netty.channel.socket.nio.NioServerBoss.select(NioServerBoss.java:163) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.NioServerBoss.run(NioServerBoss.java:42) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: org.apache.hadoop.hdfs.server.datanode.DataXceiverServer@350c3bcd sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:421) sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:249) sun.nio.ch.ServerSocketAdaptor.accept(ServerSocketAdaptor.java:113) org.apache.hadoop.hdfs.net.TcpPeerServer.accept(TcpPeerServer.java:85) org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.run(DataXceiverServer.java:145) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 17 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 40 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 18 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 36215 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 32 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #47 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: org.apache.hadoop.hdfs.server.datanode.DataXceiverServer@51e7c8f sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:421) sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:249) sun.nio.ch.ServerSocketAdaptor.accept(ServerSocketAdaptor.java:113) org.apache.hadoop.hdfs.net.TcpPeerServer.accept(TcpPeerServer.java:85) org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.run(DataXceiverServer.java:145) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 12 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 8 on default port 36903 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 8 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 44 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: DeletionService #0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 10 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 16 on default port 36903 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 37 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Container metrics unregistration java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 3 on default port 45775 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #36 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@1b08dc9e java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 15 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 17 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 47 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 15 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-56 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 17 on default port 36903 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-52 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 19 on default port 36215 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 6 on default port 36903 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50423@0x660d8b73-SendThread(127.0.0.1:50423) sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:345) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1223) Potentially hanging thread: Timer-40 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 2 on default port 34359 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/cluster_b294908c-ea89-86ee-324a-909f7c935e98/dfs/data/data6) java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:627) Potentially hanging thread: IPC Server handler 11 on default port 36903 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Thread[Thread-1327,5,FailOnTimeoutGroup] java.lang.Thread.sleep(Native Method) org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:694) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 15 on default port 36215 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 4 on default port 38931 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 7 on default port 36903 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 46447 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 9 on default port 45775 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 36 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 44107 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 37 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 26 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 22 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/213747eb-b3b4-46cc-8d04-0796ac9f80f6/cluster_b294908c-ea89-86ee-324a-909f7c935e98/dfs/data/data4/current/BP-75859831-172.31.12.81-1694032552350 java.lang.Thread.sleep(Native Method) org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:205) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 2 on default port 46281 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Client (366321005) connection to jenkins-hbase3.apache.org/172.31.12.81:41235 from jenkins java.lang.Object.wait(Native Method) org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1053) org.apache.hadoop.ipc.Client$Connection.run(Client.java:1097) Potentially hanging thread: nioEventLoopGroup-14-1 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:803) io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457) io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 27 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #41 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 11 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 1771779945@qtp-437301073-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: Listener at jenkins-hbase3.apache.org/46281 java.lang.Thread.dumpThreads(Native Method) java.lang.Thread.getAllStackTraces(Thread.java:1615) org.apache.hadoop.hbase.ResourceCheckerJUnitListener$ThreadResourceAnalyzer.getVal(ResourceCheckerJUnitListener.java:49) org.apache.hadoop.hbase.ResourceChecker.fill(ResourceChecker.java:110) org.apache.hadoop.hbase.ResourceChecker.fillEndings(ResourceChecker.java:104) org.apache.hadoop.hbase.ResourceChecker.end(ResourceChecker.java:206) org.apache.hadoop.hbase.ResourceCheckerJUnitListener.end(ResourceCheckerJUnitListener.java:165) org.apache.hadoop.hbase.ResourceCheckerJUnitListener.testFinished(ResourceCheckerJUnitListener.java:185) org.junit.runner.notification.SynchronizedRunListener.testFinished(SynchronizedRunListener.java:87) org.junit.runner.notification.RunNotifier$9.notifyListener(RunNotifier.java:225) org.junit.runner.notification.RunNotifier$SafeNotifier.run(RunNotifier.java:72) org.junit.runner.notification.RunNotifier.fireTestFinished(RunNotifier.java:222) org.junit.internal.runners.model.EachTestNotifier.fireTestFinished(EachTestNotifier.java:38) org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:372) org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) java.util.concurrent.FutureTask.run(FutureTask.java:266) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 5 on default port 36215 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-53 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 41 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 30 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: hconnection-0x1ea85617-shared-pool-17 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 45 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 34359 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #43 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: MutableQuantiles-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: 78799764@qtp-62646430-0 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:43791 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: pool-366-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 29 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-48 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 24 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 4 on default port 44107 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 21 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 1 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 9 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 48 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50423@0x660d8b73-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: IPC Server handler 2 on default port 46447 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 47 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-362-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server idle connection scanner for port 36215 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: New I/O worker #60 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 3 on default port 38931 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: ApplicationMasterLauncher #0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 5 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 1 on default port 34359 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-60 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 3 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 39 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 1 on default port 46281 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 46 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 31 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 4 on default port 45775 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 38 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-356-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #64 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 32 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 774312379@qtp-437301073-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:42999 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: IPC Server handler 23 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: region-location-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@23321065 java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 11 on default port 37405 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 4 on default port 36215 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 6 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-91703826_16 at /127.0.0.1:50128 [Receiving block BP-657318226-172.31.12.81-1694032498405:blk_1073741870_1046] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #42 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 0 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 31 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-399-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1102555593_1 at /127.0.0.1:51992 [Waiting for operation #2] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: org.apache.hadoop.hdfs.server.namenode.FSNamesystem$LazyPersistFileScrubber@3921cb59 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.server.namenode.FSNamesystem$LazyPersistFileScrubber.run(FSNamesystem.java:3975) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 6 on default port 36215 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: BP-75859831-172.31.12.81-1694032552350 heartbeating to localhost/127.0.0.1:44107 java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.IncrementalBlockReportManager.waitTillNextIBR(IncrementalBlockReportManager.java:158) org.apache.hadoop.hdfs.server.datanode.BPServiceActor.offerService(BPServiceActor.java:715) org.apache.hadoop.hdfs.server.datanode.BPServiceActor.run(BPServiceActor.java:851) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 35 on default port 40699 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-46 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: New I/O worker #56 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 12 on default port 36941 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 35304929@qtp-259296187-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor@3ed6db93 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor.run(HeartbeatManager.java:536) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1ea85617-shared-pool-15 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: pool-376-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=1259 (was 905) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=167 (was 139) - SystemLoadAverage LEAK? -, ProcessCount=170 (was 167) - ProcessCount LEAK? -, AvailableMemoryMB=2506 (was 3457) 2023-09-06T20:36:17,781 WARN [Listener at jenkins-hbase3.apache.org/46281 {}] hbase.ResourceChecker(130): Thread=1306 is superior to 500 2023-09-06T20:36:17,781 WARN [Listener at jenkins-hbase3.apache.org/46281 {}] hbase.ResourceChecker(130): OpenFileDescriptor=1259 is superior to 1024 2023-09-06T20:36:17,789 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-09-06T20:36:17,789 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5b70bf78 to 127.0.0.1:50423 2023-09-06T20:36:17,789 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:36:17,797 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-09-06T20:36:17,797 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] util.JVMClusterUtil(257): Found active master hash=128498560, stopped=false 2023-09-06T20:36:17,798 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] master.ServerManager(910): Cluster shutdown requested of master=jenkins-hbase3.apache.org,41475,1694032510566 2023-09-06T20:36:17,799 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/2/running 2023-09-06T20:36:17,799 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39115-0x100eefdf7f60005, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/2/running 2023-09-06T20:36:17,799 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] procedure2.ProcedureExecutor(630): Stopping 2023-09-06T20:36:17,799 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-09-06T20:36:17,799 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39115-0x100eefdf7f60005, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-09-06T20:36:17,800 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:36:17,800 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,39115,1694032510720' ***** 2023-09-06T20:36:17,800 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39115-0x100eefdf7f60005, quorum=127.0.0.1:50423, baseZNode=/2 Set watcher on znode that does not yet exist, /2/running 2023-09-06T20:36:17,800 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] regionserver.HRegionServer(2531): STOPPED: Shutdown requested 2023-09-06T20:36:17,800 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Set watcher on znode that does not yet exist, /2/running 2023-09-06T20:36:17,800 INFO [RS:0;jenkins-hbase3:39115 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-09-06T20:36:17,803 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-09-06T20:36:17,831 INFO [RS:0;jenkins-hbase3:39115 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@1c95151d{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-09-06T20:36:17,836 INFO [RS:0;jenkins-hbase3:39115 {}] server.AbstractConnector(383): Stopped ServerConnector@2e1a589d{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-09-06T20:36:17,836 INFO [RS:0;jenkins-hbase3:39115 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-09-06T20:36:17,838 INFO [RS:0;jenkins-hbase3:39115 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@5286da90{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-09-06T20:36:17,840 INFO [RS:0;jenkins-hbase3:39115 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@6bbaa492{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/hadoop.log.dir/,STOPPED} 2023-09-06T20:36:17,843 INFO [RS:0;jenkins-hbase3:39115 {}] regionserver.HeapMemoryManager(220): Stopping 2023-09-06T20:36:17,843 INFO [RS:0;jenkins-hbase3:39115 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2023-09-06T20:36:17,843 INFO [RS:0;jenkins-hbase3:39115 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-09-06T20:36:17,843 INFO [RS:0;jenkins-hbase3:39115 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-09-06T20:36:17,843 INFO [RS:0;jenkins-hbase3:39115 {}] regionserver.HRegionServer(3528): Received CLOSE for 932b31f1be4976500c94245ad49c9242 2023-09-06T20:36:17,845 INFO [RS:0;jenkins-hbase3:39115 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,39115,1694032510720 2023-09-06T20:36:17,845 DEBUG [RS:0;jenkins-hbase3:39115 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:36:17,846 INFO [RS:0;jenkins-hbase3:39115 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-09-06T20:36:17,846 INFO [RS:0;jenkins-hbase3:39115 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-09-06T20:36:17,846 INFO [RS:0;jenkins-hbase3:39115 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-09-06T20:36:17,846 INFO [RS:0;jenkins-hbase3:39115 {}] regionserver.HRegionServer(3528): Received CLOSE for 1588230740 2023-09-06T20:36:17,848 INFO [RS:0;jenkins-hbase3:39115 {}] regionserver.HRegionServer(1579): Waiting on 2 regions to close 2023-09-06T20:36:17,848 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 932b31f1be4976500c94245ad49c9242, disabling compactions & flushes 2023-09-06T20:36:17,848 DEBUG [RS:0;jenkins-hbase3:39115 {}] regionserver.HRegionServer(1583): Online Regions={1588230740=hbase:meta,,1.1588230740, 932b31f1be4976500c94245ad49c9242=hbase:namespace,,1694032513073.932b31f1be4976500c94245ad49c9242.} 2023-09-06T20:36:17,848 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1694032513073.932b31f1be4976500c94245ad49c9242. 2023-09-06T20:36:17,850 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1694032513073.932b31f1be4976500c94245ad49c9242. 2023-09-06T20:36:17,850 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-09-06T20:36:17,850 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1694032513073.932b31f1be4976500c94245ad49c9242. after waiting 0 ms 2023-09-06T20:36:17,850 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1694032513073.932b31f1be4976500c94245ad49c9242. 2023-09-06T20:36:17,850 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-09-06T20:36:17,850 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-09-06T20:36:17,854 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 932b31f1be4976500c94245ad49c9242 1/1 column families, dataSize=78 B heapSize=488 B 2023-09-06T20:36:17,854 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-09-06T20:36:17,854 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-09-06T20:36:17,854 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=1.26 KB heapSize=2.89 KB 2023-09-06T20:36:17,855 DEBUG [RS:0;jenkins-hbase3:39115 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 932b31f1be4976500c94245ad49c9242 2023-09-06T20:36:17,877 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/data/hbase/namespace/932b31f1be4976500c94245ad49c9242/.tmp/info/f234ee52317341cf9fa78ba46fa1c403 is 45, key is default/info:d/1694032513991/Put/seqid=0 2023-09-06T20:36:17,885 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/data/hbase/meta/1588230740/.tmp/info/63005e3916f843cc9b61359fea78575d is 143, key is hbase:namespace,,1694032513073.932b31f1be4976500c94245ad49c9242./info:regioninfo/1694032513866/Put/seqid=0 2023-09-06T20:36:17,894 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/data/hbase/namespace/932b31f1be4976500c94245ad49c9242/.tmp/info/f234ee52317341cf9fa78ba46fa1c403 2023-09-06T20:36:17,897 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.17 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/data/hbase/meta/1588230740/.tmp/info/63005e3916f843cc9b61359fea78575d 2023-09-06T20:36:17,897 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-09-06T20:36:17,924 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/data/hbase/namespace/932b31f1be4976500c94245ad49c9242/.tmp/info/f234ee52317341cf9fa78ba46fa1c403 as hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/data/hbase/namespace/932b31f1be4976500c94245ad49c9242/info/f234ee52317341cf9fa78ba46fa1c403 2023-09-06T20:36:17,931 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/data/hbase/namespace/932b31f1be4976500c94245ad49c9242/info/f234ee52317341cf9fa78ba46fa1c403, entries=2, sequenceid=6, filesize=4.9 K 2023-09-06T20:36:17,932 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 932b31f1be4976500c94245ad49c9242 in 82ms, sequenceid=6, compaction requested=false 2023-09-06T20:36:17,948 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41733/tmp/wal/data/hbase/namespace/932b31f1be4976500c94245ad49c9242/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2023-09-06T20:36:17,949 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:36:17,950 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1694032513073.932b31f1be4976500c94245ad49c9242. 2023-09-06T20:36:17,951 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 932b31f1be4976500c94245ad49c9242: 2023-09-06T20:36:17,951 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1694032513073.932b31f1be4976500c94245ad49c9242. 2023-09-06T20:36:17,951 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/data/hbase/meta/1588230740/.tmp/table/db8e00f47bdf42ea949594ea3c6cb69d is 51, key is hbase:namespace/table:state/1694032513876/Put/seqid=0 2023-09-06T20:36:18,055 DEBUG [RS:0;jenkins-hbase3:39115 {}] regionserver.HRegionServer(1609): Waiting on 1588230740 2023-09-06T20:36:18,255 DEBUG [RS:0;jenkins-hbase3:39115 {}] regionserver.HRegionServer(1609): Waiting on 1588230740 2023-09-06T20:36:18,324 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-09-06T20:36:18,324 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-09-06T20:36:18,356 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=94 B at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/data/hbase/meta/1588230740/.tmp/table/db8e00f47bdf42ea949594ea3c6cb69d 2023-09-06T20:36:18,363 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/data/hbase/meta/1588230740/.tmp/info/63005e3916f843cc9b61359fea78575d as hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/data/hbase/meta/1588230740/info/63005e3916f843cc9b61359fea78575d 2023-09-06T20:36:18,369 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/data/hbase/meta/1588230740/info/63005e3916f843cc9b61359fea78575d, entries=10, sequenceid=9, filesize=6.4 K 2023-09-06T20:36:18,370 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/data/hbase/meta/1588230740/.tmp/table/db8e00f47bdf42ea949594ea3c6cb69d as hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/data/hbase/meta/1588230740/table/db8e00f47bdf42ea949594ea3c6cb69d 2023-09-06T20:36:18,375 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/data/hbase/meta/1588230740/table/db8e00f47bdf42ea949594ea3c6cb69d, entries=2, sequenceid=9, filesize=5.1 K 2023-09-06T20:36:18,376 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~1.26 KB/1290, heapSize ~2.61 KB/2672, currentSize=0 B/0 for 1588230740 in 522ms, sequenceid=9, compaction requested=false 2023-09-06T20:36:18,392 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41733/tmp/wal/data/hbase/meta/1588230740/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2023-09-06T20:36:18,393 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:36:18,393 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-09-06T20:36:18,393 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-09-06T20:36:18,393 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-09-06T20:36:18,393 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-09-06T20:36:18,455 INFO [RS:0;jenkins-hbase3:39115 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,39115,1694032510720; all regions closed. 2023-09-06T20:36:18,464 DEBUG [RS:0;jenkins-hbase3:39115 {}] wal.AbstractFSWAL(1062): Moved 1 WAL file(s) to /tmp/wal/oldWALs 2023-09-06T20:36:18,464 INFO [RS:0;jenkins-hbase3:39115 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C39115%2C1694032510720.meta:.meta(num 1694032513004) 2023-09-06T20:36:18,471 DEBUG [RS:0;jenkins-hbase3:39115 {}] wal.AbstractFSWAL(1062): Moved 1 WAL file(s) to /tmp/wal/oldWALs 2023-09-06T20:36:18,471 INFO [RS:0;jenkins-hbase3:39115 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C39115%2C1694032510720:(num 1694032512452) 2023-09-06T20:36:18,471 DEBUG [RS:0;jenkins-hbase3:39115 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:36:18,471 INFO [RS:0;jenkins-hbase3:39115 {}] regionserver.LeaseManager(133): Closed leases 2023-09-06T20:36:18,471 INFO [RS:0;jenkins-hbase3:39115 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2023-09-06T20:36:18,471 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-09-06T20:36:18,474 INFO [RS:0;jenkins-hbase3:39115 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:39115 2023-09-06T20:36:18,481 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39115-0x100eefdf7f60005, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/2/rs/jenkins-hbase3.apache.org,39115,1694032510720 2023-09-06T20:36:18,481 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/rs 2023-09-06T20:36:18,485 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,39115,1694032510720] 2023-09-06T20:36:18,485 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,39115,1694032510720; numProcessing=1 2023-09-06T20:36:18,486 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(224): Node /2/draining/jenkins-hbase3.apache.org,39115,1694032510720 already deleted, retry=false 2023-09-06T20:36:18,486 INFO [RegionServerTracker-0 {}] master.ServerManager(569): Cluster shutdown set; jenkins-hbase3.apache.org,39115,1694032510720 expired; onlineServers=0 2023-09-06T20:36:18,486 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,41475,1694032510566' ***** 2023-09-06T20:36:18,486 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2531): STOPPED: Cluster shutdown set; onlineServer=0 2023-09-06T20:36:18,488 DEBUG [M:0;jenkins-hbase3:41475 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1955a665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-09-06T20:36:18,488 INFO [M:0;jenkins-hbase3:41475 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-09-06T20:36:18,501 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/2/master 2023-09-06T20:36:18,501 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-09-06T20:36:18,501 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Set watcher on znode that does not yet exist, /2/master 2023-09-06T20:36:18,582 INFO [M:0;jenkins-hbase3:41475 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@2ead5011{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-09-06T20:36:18,583 INFO [M:0;jenkins-hbase3:41475 {}] server.AbstractConnector(383): Stopped ServerConnector@31f32a89{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-09-06T20:36:18,583 INFO [M:0;jenkins-hbase3:41475 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-09-06T20:36:18,584 INFO [M:0;jenkins-hbase3:41475 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@1f633e42{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-09-06T20:36:18,585 INFO [M:0;jenkins-hbase3:41475 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@20cfc1d8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/hadoop.log.dir/,STOPPED} 2023-09-06T20:36:18,585 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39115-0x100eefdf7f60005, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-09-06T20:36:18,585 INFO [RS:0;jenkins-hbase3:39115 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,39115,1694032510720; zookeeper connection closed. 2023-09-06T20:36:18,585 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39115-0x100eefdf7f60005, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-09-06T20:36:18,586 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@e0df1ab {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@e0df1ab 2023-09-06T20:36:18,586 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-09-06T20:36:18,587 INFO [M:0;jenkins-hbase3:41475 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,41475,1694032510566 2023-09-06T20:36:18,587 INFO [M:0;jenkins-hbase3:41475 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,41475,1694032510566; all regions closed. 2023-09-06T20:36:18,587 DEBUG [M:0;jenkins-hbase3:41475 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:36:18,587 INFO [M:0;jenkins-hbase3:41475 {}] master.HMaster(1665): Stopping master jetty server 2023-09-06T20:36:18,588 INFO [M:0;jenkins-hbase3:41475 {}] server.AbstractConnector(383): Stopped ServerConnector@fb7fdda{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-09-06T20:36:18,588 DEBUG [M:0;jenkins-hbase3:41475 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-09-06T20:36:18,588 INFO [M:0;jenkins-hbase3:41475 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-09-06T20:36:18,588 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-09-06T20:36:18,588 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1694032512209 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1694032512209,5,FailOnTimeoutGroup] 2023-09-06T20:36:18,588 DEBUG [M:0;jenkins-hbase3:41475 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x38073e07 to 127.0.0.1:50423 2023-09-06T20:36:18,588 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1694032512210 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1694032512210,5,FailOnTimeoutGroup] 2023-09-06T20:36:18,589 DEBUG [M:0;jenkins-hbase3:41475 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:36:18,589 DEBUG [M:0;jenkins-hbase3:41475 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-09-06T20:36:18,589 INFO [M:0;jenkins-hbase3:41475 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-09-06T20:36:18,589 DEBUG [M:0;jenkins-hbase3:41475 {}] master.HMaster(1688): Stopping service threads 2023-09-06T20:36:18,589 INFO [M:0;jenkins-hbase3:41475 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-09-06T20:36:18,589 ERROR [M:0;jenkins-hbase3:41475 {}] procedure2.ProcedureExecutor(654): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] 2023-09-06T20:36:18,590 INFO [M:0;jenkins-hbase3:41475 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-09-06T20:36:18,590 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-09-06T20:36:18,590 DEBUG [M:0;jenkins-hbase3:41475 {}] zookeeper.ZKUtil(347): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Unable to get data of znode /2/master because node does not exist (not an error) 2023-09-06T20:36:18,590 WARN [M:0;jenkins-hbase3:41475 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-09-06T20:36:18,590 INFO [M:0;jenkins-hbase3:41475 {}] assignment.AssignmentManager(382): Stopping assignment manager 2023-09-06T20:36:18,591 INFO [M:0;jenkins-hbase3:41475 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-09-06T20:36:18,592 DEBUG [M:0;jenkins-hbase3:41475 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-09-06T20:36:18,608 INFO [M:0;jenkins-hbase3:41475 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:36:18,608 DEBUG [M:0;jenkins-hbase3:41475 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:36:18,608 DEBUG [M:0;jenkins-hbase3:41475 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-09-06T20:36:18,608 DEBUG [M:0;jenkins-hbase3:41475 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:36:18,608 INFO [M:0;jenkins-hbase3:41475 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=25.76 KB heapSize=33.30 KB 2023-09-06T20:36:18,623 DEBUG [M:0;jenkins-hbase3:41475 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cb8d6aa05a91442b926d4afb7e3734cd is 88, key is hbase:meta,,1/info:sn/1694032512822/Put/seqid=0 2023-09-06T20:36:19,031 INFO [M:0;jenkins-hbase3:41475 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cb8d6aa05a91442b926d4afb7e3734cd 2023-09-06T20:36:19,065 DEBUG [M:0;jenkins-hbase3:41475 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4e52022f9f0d486ea26fbe216dc7910e is 241, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1694032513059/Put/seqid=0 2023-09-06T20:36:19,482 INFO [M:0;jenkins-hbase3:41475 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25.17 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4e52022f9f0d486ea26fbe216dc7910e 2023-09-06T20:36:19,487 INFO [M:0;jenkins-hbase3:41475 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4e52022f9f0d486ea26fbe216dc7910e 2023-09-06T20:36:19,500 DEBUG [M:0;jenkins-hbase3:41475 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0679de3316a44198908dec92315baa44 is 82, key is jenkins-hbase3.apache.org,39115,1694032510720/rs:state/1694032512304/Put/seqid=0 2023-09-06T20:36:19,505 INFO [M:0;jenkins-hbase3:41475 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0679de3316a44198908dec92315baa44 2023-09-06T20:36:19,510 DEBUG [M:0;jenkins-hbase3:41475 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cb8d6aa05a91442b926d4afb7e3734cd as hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cb8d6aa05a91442b926d4afb7e3734cd 2023-09-06T20:36:19,516 INFO [M:0;jenkins-hbase3:41475 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cb8d6aa05a91442b926d4afb7e3734cd, entries=8, sequenceid=76, filesize=5.5 K 2023-09-06T20:36:19,518 DEBUG [M:0;jenkins-hbase3:41475 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4e52022f9f0d486ea26fbe216dc7910e as hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4e52022f9f0d486ea26fbe216dc7910e 2023-09-06T20:36:19,523 INFO [M:0;jenkins-hbase3:41475 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4e52022f9f0d486ea26fbe216dc7910e 2023-09-06T20:36:19,523 INFO [M:0;jenkins-hbase3:41475 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4e52022f9f0d486ea26fbe216dc7910e, entries=8, sequenceid=76, filesize=5.4 K 2023-09-06T20:36:19,524 DEBUG [M:0;jenkins-hbase3:41475 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0679de3316a44198908dec92315baa44 as hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0679de3316a44198908dec92315baa44 2023-09-06T20:36:19,528 INFO [M:0;jenkins-hbase3:41475 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41733/user/jenkins/test-data/ef7ff44d-df0a-f24b-8f20-2412d9155d8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0679de3316a44198908dec92315baa44, entries=1, sequenceid=76, filesize=5.1 K 2023-09-06T20:36:19,529 INFO [M:0;jenkins-hbase3:41475 {}] regionserver.HRegion(3022): Finished flush of dataSize ~25.76 KB/26381, heapSize ~33.01 KB/33800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 921ms, sequenceid=76, compaction requested=false 2023-09-06T20:36:19,538 INFO [M:0;jenkins-hbase3:41475 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:36:19,538 DEBUG [M:0;jenkins-hbase3:41475 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-09-06T20:36:19,542 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-09-06T20:36:19,542 INFO [M:0;jenkins-hbase3:41475 {}] flush.MasterFlushTableProcedureManager(85): stop: server shutting down. 2023-09-06T20:36:19,542 INFO [M:0;jenkins-hbase3:41475 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-09-06T20:36:19,543 INFO [M:0;jenkins-hbase3:41475 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:41475 2023-09-06T20:36:19,546 DEBUG [M:0;jenkins-hbase3:41475 {}] zookeeper.RecoverableZooKeeper(224): Node /2/rs/jenkins-hbase3.apache.org,41475,1694032510566 already deleted, retry=false 2023-09-06T20:36:19,649 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-09-06T20:36:19,649 INFO [M:0;jenkins-hbase3:41475 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,41475,1694032510566; zookeeper connection closed. 2023-09-06T20:36:19,649 DEBUG [Listener at localhost/39521-EventThread {}] zookeeper.ZKWatcher(604): master:41475-0x100eefdf7f60004, quorum=127.0.0.1:50423, baseZNode=/2 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-09-06T20:36:19,652 WARN [Listener at jenkins-hbase3.apache.org/46281 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-09-06T20:36:19,662 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-09-06T20:36:19,769 WARN [BP-577183828-172.31.12.81-1694032509166 heartbeating to localhost/127.0.0.1:41733 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-09-06T20:36:19,769 WARN [BP-577183828-172.31.12.81-1694032509166 heartbeating to localhost/127.0.0.1:41733 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-577183828-172.31.12.81-1694032509166 (Datanode Uuid f2077e27-413a-4359-9405-99f59d43a4d4) service to localhost/127.0.0.1:41733 2023-09-06T20:36:19,771 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/cluster_c0c6cff4-aaf0-dadf-2d25-742d9edde910/dfs/data/data5/current/BP-577183828-172.31.12.81-1694032509166 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-09-06T20:36:19,771 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/cluster_c0c6cff4-aaf0-dadf-2d25-742d9edde910/dfs/data/data6/current/BP-577183828-172.31.12.81-1694032509166 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-09-06T20:36:19,777 WARN [Listener at jenkins-hbase3.apache.org/46281 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-09-06T20:36:19,786 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-09-06T20:36:19,894 WARN [BP-577183828-172.31.12.81-1694032509166 heartbeating to localhost/127.0.0.1:41733 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-09-06T20:36:19,894 WARN [BP-577183828-172.31.12.81-1694032509166 heartbeating to localhost/127.0.0.1:41733 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-577183828-172.31.12.81-1694032509166 (Datanode Uuid f5c319ce-5a8a-47aa-bca3-cdfda495d7b7) service to localhost/127.0.0.1:41733 2023-09-06T20:36:19,895 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/cluster_c0c6cff4-aaf0-dadf-2d25-742d9edde910/dfs/data/data3/current/BP-577183828-172.31.12.81-1694032509166 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-09-06T20:36:19,895 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/cluster_c0c6cff4-aaf0-dadf-2d25-742d9edde910/dfs/data/data4/current/BP-577183828-172.31.12.81-1694032509166 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-09-06T20:36:19,899 WARN [Listener at jenkins-hbase3.apache.org/46281 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-09-06T20:36:19,906 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-09-06T20:36:20,012 WARN [BP-577183828-172.31.12.81-1694032509166 heartbeating to localhost/127.0.0.1:41733 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-09-06T20:36:20,013 WARN [BP-577183828-172.31.12.81-1694032509166 heartbeating to localhost/127.0.0.1:41733 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-577183828-172.31.12.81-1694032509166 (Datanode Uuid bf782b2f-52a2-4551-9a69-02d41579503c) service to localhost/127.0.0.1:41733 2023-09-06T20:36:20,013 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/cluster_c0c6cff4-aaf0-dadf-2d25-742d9edde910/dfs/data/data1/current/BP-577183828-172.31.12.81-1694032509166 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-09-06T20:36:20,014 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/fa9ed9d2-391f-28ee-40dc-4395971bf792/cluster_c0c6cff4-aaf0-dadf-2d25-742d9edde910/dfs/data/data2/current/BP-577183828-172.31.12.81-1694032509166 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-09-06T20:36:20,053 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-09-06T20:36:20,198 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-09-06T20:36:20,198 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-09-06T20:36:20,198 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-06T20:36:20,198 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3813c1aa to 127.0.0.1:50423 2023-09-06T20:36:20,198 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:36:20,198 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-09-06T20:36:20,198 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] util.JVMClusterUtil(257): Found active master hash=1490611762, stopped=false 2023-09-06T20:36:20,198 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] master.ServerManager(910): Cluster shutdown requested of master=jenkins-hbase3.apache.org,33223,1694032501568 2023-09-06T20:36:20,200 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2023-09-06T20:36:20,200 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2023-09-06T20:36:20,200 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:36:20,200 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:36:20,200 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] procedure2.ProcedureExecutor(630): Stopping 2023-09-06T20:36:20,201 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-09-06T20:36:20,201 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-09-06T20:36:20,201 DEBUG [Listener at jenkins-hbase3.apache.org/46281 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:36:20,201 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,36075,1694032504561' ***** 2023-09-06T20:36:20,201 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] regionserver.HRegionServer(2531): STOPPED: Shutdown requested 2023-09-06T20:36:20,201 INFO [RS:0;jenkins-hbase3:36075 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-09-06T20:36:20,207 INFO [RS:0;jenkins-hbase3:36075 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@48191814{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-09-06T20:36:20,207 INFO [RS:0;jenkins-hbase3:36075 {}] server.AbstractConnector(383): Stopped ServerConnector@98b5321{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-09-06T20:36:20,207 INFO [RS:0;jenkins-hbase3:36075 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-09-06T20:36:20,208 INFO [RS:0;jenkins-hbase3:36075 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@2d65fb0f{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-09-06T20:36:20,209 INFO [RS:0;jenkins-hbase3:36075 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@5023341c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/hadoop.log.dir/,STOPPED} 2023-09-06T20:36:20,210 INFO [RS:0;jenkins-hbase3:36075 {}] regionserver.HeapMemoryManager(220): Stopping 2023-09-06T20:36:20,210 INFO [RS:0;jenkins-hbase3:36075 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2023-09-06T20:36:20,210 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-09-06T20:36:20,210 INFO [RS:0;jenkins-hbase3:36075 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-09-06T20:36:20,210 INFO [RS:0;jenkins-hbase3:36075 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-09-06T20:36:20,211 INFO [RS:0;jenkins-hbase3:36075 {}] regionserver.HRegionServer(3528): Received CLOSE for c5da9ae4b22d80dadf15a6cb7c09b7e4 2023-09-06T20:36:20,211 INFO [RS:0;jenkins-hbase3:36075 {}] regionserver.HRegionServer(3528): Received CLOSE for f756fa51ee91914fe985b9228ffe9d4b 2023-09-06T20:36:20,211 INFO [RS:0;jenkins-hbase3:36075 {}] regionserver.HRegionServer(3528): Received CLOSE for cd7d22af9ee99f89485a5c716f51a037 2023-09-06T20:36:20,211 INFO [RS:0;jenkins-hbase3:36075 {}] regionserver.HRegionServer(3528): Received CLOSE for 57404e38a649f5014400023c3ebc14c3 2023-09-06T20:36:20,211 INFO [RS:0;jenkins-hbase3:36075 {}] regionserver.HRegionServer(3528): Received CLOSE for 10b75a50d5fcfa15e4a0da0ac3d85e01 2023-09-06T20:36:20,211 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing c5da9ae4b22d80dadf15a6cb7c09b7e4, disabling compactions & flushes 2023-09-06T20:36:20,211 INFO [RS:0;jenkins-hbase3:36075 {}] regionserver.HRegionServer(3528): Received CLOSE for db7e5fe8ab24349786ccf3415858edef 2023-09-06T20:36:20,211 INFO [RS:0;jenkins-hbase3:36075 {}] regionserver.HRegionServer(3528): Received CLOSE for 8e818fe31858c9c4a15e134b8245804a 2023-09-06T20:36:20,211 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region test-1694032518189,,1694032518692.c5da9ae4b22d80dadf15a6cb7c09b7e4. 2023-09-06T20:36:20,211 INFO [RS:0;jenkins-hbase3:36075 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:36:20,211 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1694032518189,,1694032518692.c5da9ae4b22d80dadf15a6cb7c09b7e4. 2023-09-06T20:36:20,211 INFO [RS:0;jenkins-hbase3:36075 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-06T20:36:20,211 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on test-1694032518189,,1694032518692.c5da9ae4b22d80dadf15a6cb7c09b7e4. after waiting 0 ms 2023-09-06T20:36:20,211 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region test-1694032518189,,1694032518692.c5da9ae4b22d80dadf15a6cb7c09b7e4. 2023-09-06T20:36:20,211 DEBUG [RS:0;jenkins-hbase3:36075 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:36:20,211 INFO [RS:0;jenkins-hbase3:36075 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-09-06T20:36:20,211 INFO [RS:0;jenkins-hbase3:36075 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-09-06T20:36:20,211 INFO [RS:0;jenkins-hbase3:36075 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-09-06T20:36:20,211 INFO [RS:0;jenkins-hbase3:36075 {}] regionserver.HRegionServer(3528): Received CLOSE for 1588230740 2023-09-06T20:36:20,212 INFO [RS:0;jenkins-hbase3:36075 {}] regionserver.HRegionServer(1579): Waiting on 8 regions to close 2023-09-06T20:36:20,214 DEBUG [RS:0;jenkins-hbase3:36075 {}] regionserver.HRegionServer(1583): Online Regions={c5da9ae4b22d80dadf15a6cb7c09b7e4=test-1694032518189,,1694032518692.c5da9ae4b22d80dadf15a6cb7c09b7e4., f756fa51ee91914fe985b9228ffe9d4b=ns4:test-16940325181893,,1694032524297.f756fa51ee91914fe985b9228ffe9d4b., cd7d22af9ee99f89485a5c716f51a037=hbase:namespace,,1694032507625.cd7d22af9ee99f89485a5c716f51a037., 57404e38a649f5014400023c3ebc14c3=backup:system_bulk,,1694032528791.57404e38a649f5014400023c3ebc14c3., 10b75a50d5fcfa15e4a0da0ac3d85e01=ns3:test-16940325181892,,1694032523174.10b75a50d5fcfa15e4a0da0ac3d85e01., 1588230740=hbase:meta,,1.1588230740, db7e5fe8ab24349786ccf3415858edef=ns2:test-16940325181891,,1694032520952.db7e5fe8ab24349786ccf3415858edef., 8e818fe31858c9c4a15e134b8245804a=backup:system,,1694032526663.8e818fe31858c9c4a15e134b8245804a.} 2023-09-06T20:36:20,215 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-09-06T20:36:20,215 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-09-06T20:36:20,215 DEBUG [RS:0;jenkins-hbase3:36075 {}] regionserver.HRegionServer(1609): Waiting on 10b75a50d5fcfa15e4a0da0ac3d85e01, 1588230740, 57404e38a649f5014400023c3ebc14c3, 8e818fe31858c9c4a15e134b8245804a, c5da9ae4b22d80dadf15a6cb7c09b7e4, cd7d22af9ee99f89485a5c716f51a037, db7e5fe8ab24349786ccf3415858edef, f756fa51ee91914fe985b9228ffe9d4b 2023-09-06T20:36:20,215 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-09-06T20:36:20,215 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-09-06T20:36:20,215 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-09-06T20:36:20,215 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=13.51 KB heapSize=24 KB 2023-09-06T20:36:20,224 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/default/test-1694032518189/c5da9ae4b22d80dadf15a6cb7c09b7e4/recovered.edits/107.seqid, newMaxSeqId=107, maxSeqId=1 2023-09-06T20:36:20,224 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:36:20,225 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed test-1694032518189,,1694032518692.c5da9ae4b22d80dadf15a6cb7c09b7e4. 2023-09-06T20:36:20,225 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for c5da9ae4b22d80dadf15a6cb7c09b7e4: 2023-09-06T20:36:20,226 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1694032518189,,1694032518692.c5da9ae4b22d80dadf15a6cb7c09b7e4. 2023-09-06T20:36:20,227 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing f756fa51ee91914fe985b9228ffe9d4b, disabling compactions & flushes 2023-09-06T20:36:20,227 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns4:test-16940325181893,,1694032524297.f756fa51ee91914fe985b9228ffe9d4b. 2023-09-06T20:36:20,227 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16940325181893,,1694032524297.f756fa51ee91914fe985b9228ffe9d4b. 2023-09-06T20:36:20,227 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16940325181893,,1694032524297.f756fa51ee91914fe985b9228ffe9d4b. after waiting 0 ms 2023-09-06T20:36:20,227 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16940325181893,,1694032524297.f756fa51ee91914fe985b9228ffe9d4b. 2023-09-06T20:36:20,230 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/ns4/test-16940325181893/f756fa51ee91914fe985b9228ffe9d4b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-09-06T20:36:20,231 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:36:20,232 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns4:test-16940325181893,,1694032524297.f756fa51ee91914fe985b9228ffe9d4b. 2023-09-06T20:36:20,232 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for f756fa51ee91914fe985b9228ffe9d4b: 2023-09-06T20:36:20,232 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-16940325181893,,1694032524297.f756fa51ee91914fe985b9228ffe9d4b. 2023-09-06T20:36:20,233 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing cd7d22af9ee99f89485a5c716f51a037, disabling compactions & flushes 2023-09-06T20:36:20,233 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1694032507625.cd7d22af9ee99f89485a5c716f51a037. 2023-09-06T20:36:20,233 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1694032507625.cd7d22af9ee99f89485a5c716f51a037. 2023-09-06T20:36:20,233 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1694032507625.cd7d22af9ee99f89485a5c716f51a037. after waiting 0 ms 2023-09-06T20:36:20,233 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1694032507625.cd7d22af9ee99f89485a5c716f51a037. 2023-09-06T20:36:20,233 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing cd7d22af9ee99f89485a5c716f51a037 1/1 column families, dataSize=249 B heapSize=1.02 KB 2023-09-06T20:36:20,242 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/hbase/meta/1588230740/.tmp/info/e718caab394b440585588dd0ec9638bc is 159, key is ns2:test-16940325181891,,1694032520952.db7e5fe8ab24349786ccf3415858edef./info:regioninfo/1694032522147/Put/seqid=0 2023-09-06T20:36:20,247 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.12 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/hbase/meta/1588230740/.tmp/info/e718caab394b440585588dd0ec9638bc 2023-09-06T20:36:20,248 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/hbase/namespace/cd7d22af9ee99f89485a5c716f51a037/.tmp/info/5a521d31ed044d9898de70c8d95a6743 is 45, key is default/info:d/1694032509010/Put/seqid=0 2023-09-06T20:36:20,254 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/hbase/namespace/cd7d22af9ee99f89485a5c716f51a037/.tmp/info/5a521d31ed044d9898de70c8d95a6743 2023-09-06T20:36:20,260 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/hbase/namespace/cd7d22af9ee99f89485a5c716f51a037/.tmp/info/5a521d31ed044d9898de70c8d95a6743 as hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/hbase/namespace/cd7d22af9ee99f89485a5c716f51a037/info/5a521d31ed044d9898de70c8d95a6743 2023-09-06T20:36:20,267 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/hbase/namespace/cd7d22af9ee99f89485a5c716f51a037/info/5a521d31ed044d9898de70c8d95a6743, entries=7, sequenceid=11, filesize=5.1 K 2023-09-06T20:36:20,268 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~249 B/249, heapSize ~1.01 KB/1032, currentSize=0 B/0 for cd7d22af9ee99f89485a5c716f51a037 in 34ms, sequenceid=11, compaction requested=false 2023-09-06T20:36:20,268 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-09-06T20:36:20,274 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/hbase/meta/1588230740/.tmp/rep_barrier/428f9954806245f0a96a0b5939374d8f is 90, key is table1,,1694032538194.f5facd6c54ed401d7afb02fa219211d1./rep_barrier:/1694032542563/DeleteFamily/seqid=0 2023-09-06T20:36:20,283 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-09-06T20:36:20,293 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/hbase/namespace/cd7d22af9ee99f89485a5c716f51a037/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2023-09-06T20:36:20,293 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:36:20,296 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1694032507625.cd7d22af9ee99f89485a5c716f51a037. 2023-09-06T20:36:20,296 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for cd7d22af9ee99f89485a5c716f51a037: 2023-09-06T20:36:20,296 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1694032507625.cd7d22af9ee99f89485a5c716f51a037. 2023-09-06T20:36:20,296 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=172 B at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/hbase/meta/1588230740/.tmp/rep_barrier/428f9954806245f0a96a0b5939374d8f 2023-09-06T20:36:20,298 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 57404e38a649f5014400023c3ebc14c3, disabling compactions & flushes 2023-09-06T20:36:20,298 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1694032528791.57404e38a649f5014400023c3ebc14c3. 2023-09-06T20:36:20,298 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1694032528791.57404e38a649f5014400023c3ebc14c3. 2023-09-06T20:36:20,298 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1694032528791.57404e38a649f5014400023c3ebc14c3. after waiting 0 ms 2023-09-06T20:36:20,298 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1694032528791.57404e38a649f5014400023c3ebc14c3. 2023-09-06T20:36:20,302 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system_bulk/57404e38a649f5014400023c3ebc14c3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-09-06T20:36:20,304 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:36:20,306 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system_bulk,,1694032528791.57404e38a649f5014400023c3ebc14c3. 2023-09-06T20:36:20,306 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 57404e38a649f5014400023c3ebc14c3: 2023-09-06T20:36:20,306 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1694032528791.57404e38a649f5014400023c3ebc14c3. 2023-09-06T20:36:20,307 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 10b75a50d5fcfa15e4a0da0ac3d85e01, disabling compactions & flushes 2023-09-06T20:36:20,308 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns3:test-16940325181892,,1694032523174.10b75a50d5fcfa15e4a0da0ac3d85e01. 2023-09-06T20:36:20,308 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16940325181892,,1694032523174.10b75a50d5fcfa15e4a0da0ac3d85e01. 2023-09-06T20:36:20,308 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16940325181892,,1694032523174.10b75a50d5fcfa15e4a0da0ac3d85e01. after waiting 0 ms 2023-09-06T20:36:20,308 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16940325181892,,1694032523174.10b75a50d5fcfa15e4a0da0ac3d85e01. 2023-09-06T20:36:20,322 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/ns3/test-16940325181892/10b75a50d5fcfa15e4a0da0ac3d85e01/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-09-06T20:36:20,323 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:36:20,325 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns3:test-16940325181892,,1694032523174.10b75a50d5fcfa15e4a0da0ac3d85e01. 2023-09-06T20:36:20,325 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 10b75a50d5fcfa15e4a0da0ac3d85e01: 2023-09-06T20:36:20,325 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-16940325181892,,1694032523174.10b75a50d5fcfa15e4a0da0ac3d85e01. 2023-09-06T20:36:20,327 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing db7e5fe8ab24349786ccf3415858edef, disabling compactions & flushes 2023-09-06T20:36:20,327 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns2:test-16940325181891,,1694032520952.db7e5fe8ab24349786ccf3415858edef. 2023-09-06T20:36:20,327 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16940325181891,,1694032520952.db7e5fe8ab24349786ccf3415858edef. 2023-09-06T20:36:20,327 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16940325181891,,1694032520952.db7e5fe8ab24349786ccf3415858edef. after waiting 0 ms 2023-09-06T20:36:20,327 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16940325181891,,1694032520952.db7e5fe8ab24349786ccf3415858edef. 2023-09-06T20:36:20,327 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing db7e5fe8ab24349786ccf3415858edef 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-09-06T20:36:20,329 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/hbase/meta/1588230740/.tmp/table/7380797b82e64309b1942eeb4c26df20 is 84, key is table1,,1694032538194.f5facd6c54ed401d7afb02fa219211d1./table:/1694032542563/DeleteFamily/seqid=0 2023-09-06T20:36:20,342 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/ns2/test-16940325181891/db7e5fe8ab24349786ccf3415858edef/.tmp/f/0997a210bdd5453d98145c1872f9f00a is 37, key is row10/f:q1/1694032523091/Put/seqid=0 2023-09-06T20:36:20,416 DEBUG [RS:0;jenkins-hbase3:36075 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 8e818fe31858c9c4a15e134b8245804a, db7e5fe8ab24349786ccf3415858edef 2023-09-06T20:36:20,550 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-09-06T20:36:20,550 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-09-06T20:36:20,616 DEBUG [RS:0;jenkins-hbase3:36075 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 8e818fe31858c9c4a15e134b8245804a, db7e5fe8ab24349786ccf3415858edef 2023-09-06T20:36:20,740 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.22 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/hbase/meta/1588230740/.tmp/table/7380797b82e64309b1942eeb4c26df20 2023-09-06T20:36:20,747 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/ns2/test-16940325181891/db7e5fe8ab24349786ccf3415858edef/.tmp/f/0997a210bdd5453d98145c1872f9f00a 2023-09-06T20:36:20,747 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/hbase/meta/1588230740/.tmp/info/e718caab394b440585588dd0ec9638bc as hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/hbase/meta/1588230740/info/e718caab394b440585588dd0ec9638bc 2023-09-06T20:36:20,753 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/ns2/test-16940325181891/db7e5fe8ab24349786ccf3415858edef/.tmp/f/0997a210bdd5453d98145c1872f9f00a as hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/ns2/test-16940325181891/db7e5fe8ab24349786ccf3415858edef/f/0997a210bdd5453d98145c1872f9f00a 2023-09-06T20:36:20,753 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/hbase/meta/1588230740/info/e718caab394b440585588dd0ec9638bc, entries=74, sequenceid=63, filesize=14.4 K 2023-09-06T20:36:20,754 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/hbase/meta/1588230740/.tmp/rep_barrier/428f9954806245f0a96a0b5939374d8f as hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/hbase/meta/1588230740/rep_barrier/428f9954806245f0a96a0b5939374d8f 2023-09-06T20:36:20,761 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/ns2/test-16940325181891/db7e5fe8ab24349786ccf3415858edef/f/0997a210bdd5453d98145c1872f9f00a, entries=99, sequenceid=103, filesize=8.2 K 2023-09-06T20:36:20,761 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/hbase/meta/1588230740/rep_barrier/428f9954806245f0a96a0b5939374d8f, entries=2, sequenceid=63, filesize=5.4 K 2023-09-06T20:36:20,762 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for db7e5fe8ab24349786ccf3415858edef in 434ms, sequenceid=103, compaction requested=false 2023-09-06T20:36:20,766 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/hbase/meta/1588230740/.tmp/table/7380797b82e64309b1942eeb4c26df20 as hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/hbase/meta/1588230740/table/7380797b82e64309b1942eeb4c26df20 2023-09-06T20:36:20,773 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/ns2/test-16940325181891/db7e5fe8ab24349786ccf3415858edef/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-09-06T20:36:20,773 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:36:20,775 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns2:test-16940325181891,,1694032520952.db7e5fe8ab24349786ccf3415858edef. 2023-09-06T20:36:20,776 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for db7e5fe8ab24349786ccf3415858edef: 2023-09-06T20:36:20,776 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-16940325181891,,1694032520952.db7e5fe8ab24349786ccf3415858edef. 2023-09-06T20:36:20,776 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 8e818fe31858c9c4a15e134b8245804a, disabling compactions & flushes 2023-09-06T20:36:20,776 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system,,1694032526663.8e818fe31858c9c4a15e134b8245804a. 2023-09-06T20:36:20,776 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1694032526663.8e818fe31858c9c4a15e134b8245804a. 2023-09-06T20:36:20,776 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1694032526663.8e818fe31858c9c4a15e134b8245804a. after waiting 0 ms 2023-09-06T20:36:20,776 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1694032526663.8e818fe31858c9c4a15e134b8245804a. 2023-09-06T20:36:20,776 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 8e818fe31858c9c4a15e134b8245804a 2/2 column families, dataSize=985 B heapSize=1.97 KB 2023-09-06T20:36:20,784 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/hbase/meta/1588230740/table/7380797b82e64309b1942eeb4c26df20, entries=17, sequenceid=63, filesize=6.0 K 2023-09-06T20:36:20,784 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~13.51 KB/13834, heapSize ~23.95 KB/24528, currentSize=0 B/0 for 1588230740 in 569ms, sequenceid=63, compaction requested=false 2023-09-06T20:36:20,785 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-09-06T20:36:20,793 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/.tmp/meta/f6569a9ad4614e1fa7983629d905f6a8 is 169, key is trslm:hdfs://localhost:41733/backupUT\x00test-1694032518189/meta:log-roll-map/1694032551889/Put/seqid=0 2023-09-06T20:36:20,800 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/hbase/meta/1588230740/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=1 2023-09-06T20:36:20,800 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:36:20,801 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-09-06T20:36:20,802 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-09-06T20:36:20,802 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-09-06T20:36:20,802 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-09-06T20:36:20,816 DEBUG [RS:0;jenkins-hbase3:36075 {}] regionserver.HRegionServer(1609): Waiting on 8e818fe31858c9c4a15e134b8245804a 2023-09-06T20:36:21,016 DEBUG [RS:0;jenkins-hbase3:36075 {}] regionserver.HRegionServer(1609): Waiting on 8e818fe31858c9c4a15e134b8245804a 2023-09-06T20:36:21,202 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=27 (bloomFilter=true), to=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/.tmp/meta/f6569a9ad4614e1fa7983629d905f6a8 2023-09-06T20:36:21,211 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/.tmp/session/8c017f6c0eab4c16a5450c346daba4fd is 310, key is session:backup_1694032542698/session:context/1694032552313/Put/seqid=0 2023-09-06T20:36:21,215 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=534 B at sequenceid=27 (bloomFilter=true), to=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/.tmp/session/8c017f6c0eab4c16a5450c346daba4fd 2023-09-06T20:36:21,216 INFO [RS:0;jenkins-hbase3:36075 {}] regionserver.HRegionServer(1579): Waiting on 1 regions to close 2023-09-06T20:36:21,216 DEBUG [RS:0;jenkins-hbase3:36075 {}] regionserver.HRegionServer(1583): Online Regions={8e818fe31858c9c4a15e134b8245804a=backup:system,,1694032526663.8e818fe31858c9c4a15e134b8245804a.} 2023-09-06T20:36:21,217 DEBUG [RS:0;jenkins-hbase3:36075 {}] regionserver.HRegionServer(1609): Waiting on 8e818fe31858c9c4a15e134b8245804a 2023-09-06T20:36:21,221 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/.tmp/meta/f6569a9ad4614e1fa7983629d905f6a8 as hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/meta/f6569a9ad4614e1fa7983629d905f6a8 2023-09-06T20:36:21,226 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/meta/f6569a9ad4614e1fa7983629d905f6a8, entries=4, sequenceid=27, filesize=5.5 K 2023-09-06T20:36:21,227 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/.tmp/session/8c017f6c0eab4c16a5450c346daba4fd as hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/session/8c017f6c0eab4c16a5450c346daba4fd 2023-09-06T20:36:21,233 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/session/8c017f6c0eab4c16a5450c346daba4fd, entries=2, sequenceid=27, filesize=5.3 K 2023-09-06T20:36:21,234 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~985 B/985, heapSize ~1.94 KB/1984, currentSize=0 B/0 for 8e818fe31858c9c4a15e134b8245804a in 457ms, sequenceid=27, compaction requested=true 2023-09-06T20:36:21,257 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/data/backup/system/8e818fe31858c9c4a15e134b8245804a/recovered.edits/30.seqid, newMaxSeqId=30, maxSeqId=1 2023-09-06T20:36:21,258 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:36:21,259 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system,,1694032526663.8e818fe31858c9c4a15e134b8245804a. 2023-09-06T20:36:21,259 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 8e818fe31858c9c4a15e134b8245804a: 2023-09-06T20:36:21,259 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1694032526663.8e818fe31858c9c4a15e134b8245804a. 2023-09-06T20:36:21,417 INFO [RS:0;jenkins-hbase3:36075 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,36075,1694032504561; all regions closed. 2023-09-06T20:36:21,425 DEBUG [RS:0;jenkins-hbase3:36075 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/oldWALs 2023-09-06T20:36:21,425 INFO [RS:0;jenkins-hbase3:36075 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C36075%2C1694032504561.meta:.meta(num 1694032546912) 2023-09-06T20:36:21,429 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/WALs/jenkins-hbase3.apache.org,36075,1694032504561/jenkins-hbase3.apache.org%2C36075%2C1694032504561.1694032546866 not finished, retry = 0 2023-09-06T20:36:21,533 DEBUG [RS:0;jenkins-hbase3:36075 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/oldWALs 2023-09-06T20:36:21,533 INFO [RS:0;jenkins-hbase3:36075 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C36075%2C1694032504561:(num 1694032546866) 2023-09-06T20:36:21,533 DEBUG [RS:0;jenkins-hbase3:36075 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:36:21,533 INFO [RS:0;jenkins-hbase3:36075 {}] regionserver.LeaseManager(133): Closed leases 2023-09-06T20:36:21,533 INFO [RS:0;jenkins-hbase3:36075 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2023-09-06T20:36:21,534 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-09-06T20:36:21,536 INFO [RS:0;jenkins-hbase3:36075 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:36075 2023-09-06T20:36:21,542 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-09-06T20:36:21,542 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rs/jenkins-hbase3.apache.org,36075,1694032504561 2023-09-06T20:36:21,548 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,36075,1694032504561] 2023-09-06T20:36:21,549 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,36075,1694032504561; numProcessing=1 2023-09-06T20:36:21,550 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(224): Node /1/draining/jenkins-hbase3.apache.org,36075,1694032504561 already deleted, retry=false 2023-09-06T20:36:21,550 INFO [RegionServerTracker-0 {}] master.ServerManager(569): Cluster shutdown set; jenkins-hbase3.apache.org,36075,1694032504561 expired; onlineServers=0 2023-09-06T20:36:21,550 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,33223,1694032501568' ***** 2023-09-06T20:36:21,550 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2531): STOPPED: Cluster shutdown set; onlineServer=0 2023-09-06T20:36:21,551 DEBUG [M:0;jenkins-hbase3:33223 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d8ab1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-09-06T20:36:21,551 INFO [M:0;jenkins-hbase3:33223 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-09-06T20:36:21,568 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/master 2023-09-06T20:36:21,568 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:36:21,578 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-09-06T20:36:21,585 INFO [M:0;jenkins-hbase3:33223 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@25a564df{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-09-06T20:36:21,585 INFO [M:0;jenkins-hbase3:33223 {}] server.AbstractConnector(383): Stopped ServerConnector@4420f3e7{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-09-06T20:36:21,585 INFO [M:0;jenkins-hbase3:33223 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-09-06T20:36:21,587 INFO [M:0;jenkins-hbase3:33223 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@4c63d339{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-09-06T20:36:21,588 INFO [M:0;jenkins-hbase3:33223 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@24da0689{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/hadoop.log.dir/,STOPPED} 2023-09-06T20:36:21,589 INFO [M:0;jenkins-hbase3:33223 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,33223,1694032501568 2023-09-06T20:36:21,589 INFO [M:0;jenkins-hbase3:33223 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,33223,1694032501568; all regions closed. 2023-09-06T20:36:21,589 DEBUG [M:0;jenkins-hbase3:33223 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:36:21,589 INFO [M:0;jenkins-hbase3:33223 {}] master.HMaster(1665): Stopping master jetty server 2023-09-06T20:36:21,590 INFO [M:0;jenkins-hbase3:33223 {}] server.AbstractConnector(383): Stopped ServerConnector@68792fe9{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-09-06T20:36:21,590 DEBUG [M:0;jenkins-hbase3:33223 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-09-06T20:36:21,590 INFO [M:0;jenkins-hbase3:33223 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-09-06T20:36:21,590 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-09-06T20:36:21,591 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1694032506325 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1694032506325,5,FailOnTimeoutGroup] 2023-09-06T20:36:21,591 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1694032506326 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1694032506326,5,FailOnTimeoutGroup] 2023-09-06T20:36:21,591 DEBUG [M:0;jenkins-hbase3:33223 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7f1b5456 to 127.0.0.1:50423 2023-09-06T20:36:21,591 DEBUG [M:0;jenkins-hbase3:33223 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:36:21,591 DEBUG [M:0;jenkins-hbase3:33223 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-09-06T20:36:21,591 INFO [M:0;jenkins-hbase3:33223 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-09-06T20:36:21,591 DEBUG [M:0;jenkins-hbase3:33223 {}] master.HMaster(1688): Stopping service threads 2023-09-06T20:36:21,591 INFO [M:0;jenkins-hbase3:33223 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-09-06T20:36:21,591 ERROR [M:0;jenkins-hbase3:33223 {}] procedure2.ProcedureExecutor(654): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-2,5,PEWorkerGroup] 2023-09-06T20:36:21,591 INFO [M:0;jenkins-hbase3:33223 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-09-06T20:36:21,592 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-09-06T20:36:21,592 DEBUG [M:0;jenkins-hbase3:33223 {}] zookeeper.ZKUtil(347): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Unable to get data of znode /1/master because node does not exist (not an error) 2023-09-06T20:36:21,592 WARN [M:0;jenkins-hbase3:33223 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-09-06T20:36:21,592 INFO [M:0;jenkins-hbase3:33223 {}] assignment.AssignmentManager(382): Stopping assignment manager 2023-09-06T20:36:21,592 INFO [M:0;jenkins-hbase3:33223 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-09-06T20:36:21,593 DEBUG [M:0;jenkins-hbase3:33223 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-09-06T20:36:21,605 INFO [M:0;jenkins-hbase3:33223 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:36:21,605 DEBUG [M:0;jenkins-hbase3:33223 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:36:21,605 DEBUG [M:0;jenkins-hbase3:33223 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-09-06T20:36:21,605 DEBUG [M:0;jenkins-hbase3:33223 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:36:21,605 INFO [M:0;jenkins-hbase3:33223 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=203.19 KB heapSize=244.16 KB 2023-09-06T20:36:21,621 DEBUG [M:0;jenkins-hbase3:33223 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9c3a5770a75946a8b0f2170274463435 is 88, key is hbase:meta,,1/info:sn/1694032506952/Put/seqid=0 2023-09-06T20:36:21,625 INFO [M:0;jenkins-hbase3:33223 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9c3a5770a75946a8b0f2170274463435 2023-09-06T20:36:21,643 DEBUG [M:0;jenkins-hbase3:33223 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/27b48dfdddd542dd9845846fa11f23b5 is 987, key is \x00\x00\x00\x00\x00\x00\x00\x1D/proc:d/1694032529581/Put/seqid=0 2023-09-06T20:36:21,649 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-09-06T20:36:21,649 INFO [RS:0;jenkins-hbase3:36075 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,36075,1694032504561; zookeeper connection closed. 2023-09-06T20:36:21,649 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36075-0x100eefdf7f60001, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-09-06T20:36:21,649 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@518f9a98 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@518f9a98 2023-09-06T20:36:21,649 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-09-06T20:36:22,047 INFO [M:0;jenkins-hbase3:33223 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=202.59 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/27b48dfdddd542dd9845846fa11f23b5 2023-09-06T20:36:22,052 INFO [M:0;jenkins-hbase3:33223 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 27b48dfdddd542dd9845846fa11f23b5 2023-09-06T20:36:22,065 DEBUG [M:0;jenkins-hbase3:33223 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/230a71e917344f7c8ac65b04b8f9a6fc is 82, key is jenkins-hbase3.apache.org,36075,1694032504561/rs:state/1694032506464/Put/seqid=0 2023-09-06T20:36:22,172 INFO [Socket Reader #1 for port 0 {}] ipc.Server$Connection(2001): Auth successful for appattempt_1694032555510_0001_000001 (auth:SIMPLE) 2023-09-06T20:36:22,470 INFO [M:0;jenkins-hbase3:33223 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/230a71e917344f7c8ac65b04b8f9a6fc 2023-09-06T20:36:22,476 DEBUG [M:0;jenkins-hbase3:33223 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9c3a5770a75946a8b0f2170274463435 as hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9c3a5770a75946a8b0f2170274463435 2023-09-06T20:36:22,481 INFO [M:0;jenkins-hbase3:33223 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9c3a5770a75946a8b0f2170274463435, entries=8, sequenceid=480, filesize=5.5 K 2023-09-06T20:36:22,483 DEBUG [M:0;jenkins-hbase3:33223 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/27b48dfdddd542dd9845846fa11f23b5 as hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/27b48dfdddd542dd9845846fa11f23b5 2023-09-06T20:36:22,489 INFO [M:0;jenkins-hbase3:33223 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 27b48dfdddd542dd9845846fa11f23b5 2023-09-06T20:36:22,489 INFO [M:0;jenkins-hbase3:33223 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/27b48dfdddd542dd9845846fa11f23b5, entries=53, sequenceid=480, filesize=16.7 K 2023-09-06T20:36:22,490 DEBUG [M:0;jenkins-hbase3:33223 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/230a71e917344f7c8ac65b04b8f9a6fc as hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/230a71e917344f7c8ac65b04b8f9a6fc 2023-09-06T20:36:22,495 INFO [M:0;jenkins-hbase3:33223 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36869/user/jenkins/test-data/95d0e060-37bc-0445-8236-42b7e89407dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/230a71e917344f7c8ac65b04b8f9a6fc, entries=1, sequenceid=480, filesize=5.1 K 2023-09-06T20:36:22,496 INFO [M:0;jenkins-hbase3:33223 {}] regionserver.HRegion(3022): Finished flush of dataSize ~203.19 KB/208065, heapSize ~243.86 KB/249712, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 891ms, sequenceid=480, compaction requested=false 2023-09-06T20:36:22,525 INFO [M:0;jenkins-hbase3:33223 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:36:22,525 DEBUG [M:0;jenkins-hbase3:33223 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-09-06T20:36:22,551 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-09-06T20:36:22,551 INFO [M:0;jenkins-hbase3:33223 {}] flush.MasterFlushTableProcedureManager(85): stop: server shutting down. 2023-09-06T20:36:22,551 INFO [M:0;jenkins-hbase3:33223 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-09-06T20:36:22,553 INFO [M:0;jenkins-hbase3:33223 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:33223 2023-09-06T20:36:22,554 DEBUG [M:0;jenkins-hbase3:33223 {}] zookeeper.RecoverableZooKeeper(224): Node /1/rs/jenkins-hbase3.apache.org,33223,1694032501568 already deleted, retry=false 2023-09-06T20:36:22,670 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-09-06T20:36:22,670 INFO [M:0;jenkins-hbase3:33223 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,33223,1694032501568; zookeeper connection closed. 2023-09-06T20:36:22,671 DEBUG [Listener at localhost/43069-EventThread {}] zookeeper.ZKWatcher(604): master:33223-0x100eefdf7f60000, quorum=127.0.0.1:50423, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-09-06T20:36:22,683 WARN [Listener at jenkins-hbase3.apache.org/46281 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-09-06T20:36:22,689 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-09-06T20:36:22,790 WARN [BP-657318226-172.31.12.81-1694032498405 heartbeating to localhost/127.0.0.1:36869 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-09-06T20:36:22,790 WARN [BP-657318226-172.31.12.81-1694032498405 heartbeating to localhost/127.0.0.1:36869 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-657318226-172.31.12.81-1694032498405 (Datanode Uuid 8043810c-8db8-439d-b251-34c1f12b8a82) service to localhost/127.0.0.1:36869 2023-09-06T20:36:22,791 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/cluster_8944b83c-4af0-4d3a-395e-af5ae00b91a1/dfs/data/data1/current/BP-657318226-172.31.12.81-1694032498405 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-09-06T20:36:22,791 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/4d5393f1-fae7-4928-4dba-a3ebc5fc4930/cluster_8944b83c-4af0-4d3a-395e-af5ae00b91a1/dfs/data/data2/current/BP-657318226-172.31.12.81-1694032498405 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-09-06T20:36:22,847 INFO [Finalizer {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-06T20:36:22,847 DEBUG [Finalizer {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x660d8b73 to 127.0.0.1:50423 2023-09-06T20:36:22,847 DEBUG [Finalizer {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:36:22,860 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-09-06T20:36:22,878 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2023-09-06T20:36:22,912 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-09-06T20:36:22,913 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] hbase.HBaseTestingUtility(2857): Stopping mini mapreduce cluster... 2023-09-06T20:36:22,926 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-09-06T20:36:23,049 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-09-06T20:36:23,175 ERROR [Thread[Thread-493,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-09-06T20:36:23,176 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-09-06T20:36:23,184 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2023-09-06T20:36:23,193 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(61): Returning, interrupted : java.lang.InterruptedException 2023-09-06T20:36:23,194 ERROR [Thread[Thread-503,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-09-06T20:36:23,205 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-09-06T20:36:23,222 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:36:23,298 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-09-06T20:36:23,299 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-09-06T20:36:23,299 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16940325181893 2023-09-06T20:36:23,299 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-09-06T20:36:23,299 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1694032518189 2023-09-06T20:36:23,299 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-09-06T20:36:23,299 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-09-06T20:36:23,299 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-09-06T20:36:23,299 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16940325181892 2023-09-06T20:36:23,299 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:36:23,299 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16940325181891 2023-09-06T20:36:23,299 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2023-09-06T20:36:23,310 ERROR [Thread[Thread-468,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-09-06T20:36:23,310 INFO [Listener at jenkins-hbase3.apache.org/46281 {}] hbase.HBaseTestingUtility(2860): Mini mapreduce cluster stopped