2023-10-02 12:32:41,887 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@13c9d689 2023-10-02 12:32:41,903 main DEBUG Took 0.013125 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-10-02 12:32:41,904 main DEBUG PluginManager 'Core' found 129 plugins 2023-10-02 12:32:41,904 main DEBUG PluginManager 'Level' found 0 plugins 2023-10-02 12:32:41,905 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-10-02 12:32:41,906 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:32:41,916 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-10-02 12:32:41,933 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:32:41,936 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:32:41,936 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:32:41,937 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:32:41,937 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:32:41,938 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:32:41,939 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:32:41,939 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:32:41,940 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:32:41,940 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:32:41,941 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:32:41,941 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:32:41,942 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:32:41,942 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:32:41,943 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:32:41,943 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:32:41,944 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:32:41,944 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:32:41,945 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:32:41,945 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:32:41,946 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:32:41,946 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:32:41,947 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:32:41,947 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:32:41,947 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:32:41,948 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-10-02 12:32:41,951 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:32:41,953 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-10-02 12:32:41,956 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-10-02 12:32:41,956 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-10-02 12:32:41,957 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-10-02 12:32:41,958 main DEBUG PluginManager 'Converter' found 47 plugins 2023-10-02 12:32:41,975 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-10-02 12:32:41,979 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-10-02 12:32:41,984 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-10-02 12:32:41,985 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-10-02 12:32:41,985 main DEBUG createAppenders(={Console}) 2023-10-02 12:32:41,986 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@13c9d689 initialized 2023-10-02 12:32:41,987 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@13c9d689 2023-10-02 12:32:41,987 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@13c9d689 OK. 2023-10-02 12:32:41,988 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-10-02 12:32:41,988 main DEBUG OutputStream closed 2023-10-02 12:32:41,989 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-10-02 12:32:41,989 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-10-02 12:32:41,989 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@15c43bd9 OK 2023-10-02 12:32:42,046 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-10-02 12:32:42,048 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-10-02 12:32:42,050 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-10-02 12:32:42,051 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-10-02 12:32:42,051 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-10-02 12:32:42,052 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-10-02 12:32:42,052 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-10-02 12:32:42,052 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-10-02 12:32:42,052 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-10-02 12:32:42,053 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-10-02 12:32:42,053 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-10-02 12:32:42,053 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-10-02 12:32:42,054 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-10-02 12:32:42,054 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-10-02 12:32:42,054 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-10-02 12:32:42,054 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-10-02 12:32:42,055 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-10-02 12:32:42,056 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-10-02 12:32:42,059 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-10-02 12:32:42,059 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@1b6e1eff) with optional ClassLoader: null 2023-10-02 12:32:42,059 main DEBUG Shutdown hook enabled. Registering a new one. 2023-10-02 12:32:42,060 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@1b6e1eff] started OK. 2023-10-02T12:32:42,092 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestRemoteRestore timeout: 13 mins 2023-10-02 12:32:42,096 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-10-02 12:32:42,097 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-10-02T12:32:42,488 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a 2023-10-02T12:32:42,493 DEBUG [Time-limited test {}] impl.BackupManager(127): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-10-02T12:32:42,495 DEBUG [Time-limited test {}] impl.BackupManager(157): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:32:42,497 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-10-02T12:32:42,502 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/cluster_f3ba57df-0a4a-7a4e-2411-dac870a6ebff, deleteOnExit=true 2023-10-02T12:32:42,502 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-10-02T12:32:42,503 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/test.cache.data in system properties and HBase conf 2023-10-02T12:32:42,503 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/hadoop.tmp.dir in system properties and HBase conf 2023-10-02T12:32:42,503 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/hadoop.log.dir in system properties and HBase conf 2023-10-02T12:32:42,504 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/mapreduce.cluster.local.dir in system properties and HBase conf 2023-10-02T12:32:42,504 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-10-02T12:32:42,504 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-10-02T12:32:42,630 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-10-02T12:32:43,164 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-10-02T12:32:43,169 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-10-02T12:32:43,169 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-10-02T12:32:43,170 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-10-02T12:32:43,170 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-02T12:32:43,170 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-10-02T12:32:43,171 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-10-02T12:32:43,171 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-02T12:32:43,171 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-02T12:32:43,171 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-10-02T12:32:43,172 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/nfs.dump.dir in system properties and HBase conf 2023-10-02T12:32:43,172 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/java.io.tmpdir in system properties and HBase conf 2023-10-02T12:32:43,172 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-02T12:32:43,173 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-10-02T12:32:43,173 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-10-02T12:32:43,786 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-10-02T12:32:43,791 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-10-02T12:32:44,098 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-10-02T12:32:44,235 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@270d40fe via org.mortbay.log.Slf4jLog 2023-10-02T12:32:44,256 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:32:44,307 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:32:44,344 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/java.io.tmpdir/Jetty_localhost_33661_hdfs____.tttz5x/webapp 2023-10-02T12:32:44,485 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:33661 2023-10-02T12:32:44,513 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-10-02T12:32:44,514 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-10-02T12:32:45,137 WARN [Listener at localhost/45521 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-10-02T12:32:45,206 WARN [Listener at localhost/45521 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-10-02T12:32:45,225 WARN [Listener at localhost/45521 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:32:45,231 INFO [Listener at localhost/45521 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:32:45,237 INFO [Listener at localhost/45521 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/java.io.tmpdir/Jetty_localhost_40771_datanode____.hrygjo/webapp 2023-10-02T12:32:45,335 INFO [Listener at localhost/45521 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:40771 2023-10-02T12:32:45,645 WARN [Listener at localhost/35185 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-10-02T12:32:48,029 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-10-02T12:32:48,191 DEBUG [Listener at localhost/35185 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a 2023-10-02T12:32:48,193 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xdcde2381587a2721: Processing first storage report for DS-c62d647f-3988-4c56-a813-dcabca820155 from datanode 9eb68d63-ba42-497c-97b5-303b9b9c3b18 2023-10-02T12:32:48,195 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xdcde2381587a2721: from storage DS-c62d647f-3988-4c56-a813-dcabca820155 node DatanodeRegistration(127.0.0.1:43099, datanodeUuid=9eb68d63-ba42-497c-97b5-303b9b9c3b18, infoPort=44019, infoSecurePort=0, ipcPort=35185, storageInfo=lv=-57;cid=testClusterID;nsid=1483520982;c=1696249963873), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2023-10-02T12:32:48,195 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xdcde2381587a2721: Processing first storage report for DS-2a8ee18a-6123-4665-8534-48771ab28996 from datanode 9eb68d63-ba42-497c-97b5-303b9b9c3b18 2023-10-02T12:32:48,195 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xdcde2381587a2721: from storage DS-2a8ee18a-6123-4665-8534-48771ab28996 node DatanodeRegistration(127.0.0.1:43099, datanodeUuid=9eb68d63-ba42-497c-97b5-303b9b9c3b18, infoPort=44019, infoSecurePort=0, ipcPort=35185, storageInfo=lv=-57;cid=testClusterID;nsid=1483520982;c=1696249963873), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-10-02T12:32:48,296 INFO [Listener at localhost/35185 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/cluster_f3ba57df-0a4a-7a4e-2411-dac870a6ebff/zookeeper_0, clientPort=49524, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/cluster_f3ba57df-0a4a-7a4e-2411-dac870a6ebff/zookeeper_0/version-2, dataDirSize=424 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/cluster_f3ba57df-0a4a-7a4e-2411-dac870a6ebff/zookeeper_0/version-2, dataLogSize=424 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, serverId=0 2023-10-02T12:32:48,314 INFO [Listener at localhost/35185 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=49524 2023-10-02T12:32:48,321 INFO [Listener at localhost/35185 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:32:48,323 INFO [Listener at localhost/35185 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:32:48,978 INFO [Listener at localhost/35185 {}] util.FSUtils(462): Created version file at hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560 with version=8 2023-10-02T12:32:48,979 INFO [Listener at localhost/35185 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/hbase-staging 2023-10-02T12:32:48,996 DEBUG [Listener at localhost/35185 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-10-02T12:32:48,996 DEBUG [Listener at localhost/35185 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-10-02T12:32:48,996 DEBUG [Listener at localhost/35185 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-10-02T12:32:48,996 DEBUG [Listener at localhost/35185 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-10-02T12:32:49,216 DEBUG [Listener at localhost/35185 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-10-02T12:32:50,004 INFO [Listener at localhost/35185 {}] client.ConnectionUtils(128): master/jenkins-hbase3:0 server-side Connection retries=45 2023-10-02T12:32:50,053 INFO [Listener at localhost/35185 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-02T12:32:50,054 INFO [Listener at localhost/35185 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-10-02T12:32:50,054 INFO [Listener at localhost/35185 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-10-02T12:32:50,054 INFO [Listener at localhost/35185 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-02T12:32:50,054 INFO [Listener at localhost/35185 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-10-02T12:32:50,213 INFO [Listener at localhost/35185 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-10-02T12:32:50,361 INFO [Listener at localhost/35185 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-10-02T12:32:50,388 DEBUG [Listener at localhost/35185 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-10-02T12:32:50,396 INFO [Listener at localhost/35185 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-10-02T12:32:50,469 DEBUG [Listener at localhost/35185 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 27987 (auto-detected) 2023-10-02T12:32:50,470 DEBUG [Listener at localhost/35185 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:1c:ff:fe:e2:13:ea (auto-detected) 2023-10-02T12:32:50,512 INFO [Listener at localhost/35185 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:45341 2023-10-02T12:32:50,532 INFO [Listener at localhost/35185 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:32:50,535 INFO [Listener at localhost/35185 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:32:50,560 INFO [Listener at localhost/35185 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:45341 connecting to ZooKeeper ensemble=127.0.0.1:49524 2023-10-02T12:32:50,613 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:453410x0, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-10-02T12:32:50,617 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:45341-0x1017329d6f10000 connected 2023-10-02T12:32:50,653 DEBUG [Listener at localhost/35185 {}] zookeeper.ZKUtil(113): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-10-02T12:32:50,657 DEBUG [Listener at localhost/35185 {}] zookeeper.ZKUtil(113): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-10-02T12:32:50,661 DEBUG [Listener at localhost/35185 {}] zookeeper.ZKUtil(113): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-10-02T12:32:50,672 DEBUG [Listener at localhost/35185 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45341 2023-10-02T12:32:50,673 DEBUG [Listener at localhost/35185 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45341 2023-10-02T12:32:50,673 DEBUG [Listener at localhost/35185 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45341 2023-10-02T12:32:50,674 DEBUG [Listener at localhost/35185 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45341 2023-10-02T12:32:50,674 DEBUG [Listener at localhost/35185 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45341 2023-10-02T12:32:50,732 INFO [Listener at localhost/35185 {}] log.Log(170): Logging initialized @9572ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2023-10-02T12:32:50,865 INFO [Listener at localhost/35185 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-10-02T12:32:50,866 INFO [Listener at localhost/35185 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-10-02T12:32:50,867 INFO [Listener at localhost/35185 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-10-02T12:32:50,869 INFO [Listener at localhost/35185 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-10-02T12:32:50,869 INFO [Listener at localhost/35185 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-10-02T12:32:50,869 INFO [Listener at localhost/35185 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-10-02T12:32:50,873 INFO [Listener at localhost/35185 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-10-02T12:32:51,028 INFO [Listener at localhost/35185 {}] http.HttpServer(1219): Jetty bound to port 45493 2023-10-02T12:32:51,031 INFO [Listener at localhost/35185 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-10-02T12:32:51,072 INFO [Listener at localhost/35185 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:32:51,077 INFO [Listener at localhost/35185 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@7d8f62fb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/hadoop.log.dir/,AVAILABLE} 2023-10-02T12:32:51,077 INFO [Listener at localhost/35185 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:32:51,077 INFO [Listener at localhost/35185 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@5eae4c46{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-10-02T12:32:51,143 INFO [Listener at localhost/35185 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-10-02T12:32:51,158 INFO [Listener at localhost/35185 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-10-02T12:32:51,158 INFO [Listener at localhost/35185 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-10-02T12:32:51,160 INFO [Listener at localhost/35185 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-10-02T12:32:51,168 INFO [Listener at localhost/35185 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:32:52,438 INFO [Listener at localhost/35185 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@11de04f5{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-10-02T12:32:52,450 INFO [Listener at localhost/35185 {}] server.AbstractConnector(333): Started ServerConnector@3c70a669{HTTP/1.1, (http/1.1)}{0.0.0.0:45493} 2023-10-02T12:32:52,450 INFO [Listener at localhost/35185 {}] server.Server(415): Started @11290ms 2023-10-02T12:32:52,454 INFO [Listener at localhost/35185 {}] master.HMaster(485): hbase.rootdir=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560, hbase.cluster.distributed=false 2023-10-02T12:32:52,522 INFO [Listener at localhost/35185 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase3:0 server-side Connection retries=45 2023-10-02T12:32:52,522 INFO [Listener at localhost/35185 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-02T12:32:52,522 INFO [Listener at localhost/35185 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-10-02T12:32:52,523 INFO [Listener at localhost/35185 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-10-02T12:32:52,523 INFO [Listener at localhost/35185 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-02T12:32:52,523 INFO [Listener at localhost/35185 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-10-02T12:32:52,535 INFO [Listener at localhost/35185 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-10-02T12:32:52,537 INFO [Listener at localhost/35185 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-10-02T12:32:52,539 INFO [Listener at localhost/35185 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:38375 2023-10-02T12:32:52,541 INFO [Listener at localhost/35185 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-10-02T12:32:52,549 DEBUG [Listener at localhost/35185 {}] mob.MobFileCache(121): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-10-02T12:32:52,550 INFO [Listener at localhost/35185 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:32:52,553 INFO [Listener at localhost/35185 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:32:52,556 INFO [Listener at localhost/35185 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:38375 connecting to ZooKeeper ensemble=127.0.0.1:49524 2023-10-02T12:32:52,560 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): regionserver:383750x0, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-10-02T12:32:52,561 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:38375-0x1017329d6f10001 connected 2023-10-02T12:32:52,561 DEBUG [Listener at localhost/35185 {}] zookeeper.ZKUtil(113): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-10-02T12:32:52,562 DEBUG [Listener at localhost/35185 {}] zookeeper.ZKUtil(113): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-10-02T12:32:52,563 DEBUG [Listener at localhost/35185 {}] zookeeper.ZKUtil(113): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-10-02T12:32:52,564 DEBUG [Listener at localhost/35185 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38375 2023-10-02T12:32:52,566 DEBUG [Listener at localhost/35185 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38375 2023-10-02T12:32:52,566 DEBUG [Listener at localhost/35185 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38375 2023-10-02T12:32:52,570 DEBUG [Listener at localhost/35185 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38375 2023-10-02T12:32:52,570 DEBUG [Listener at localhost/35185 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38375 2023-10-02T12:32:52,575 INFO [Listener at localhost/35185 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-10-02T12:32:52,575 INFO [Listener at localhost/35185 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-10-02T12:32:52,575 INFO [Listener at localhost/35185 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-10-02T12:32:52,577 INFO [Listener at localhost/35185 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-10-02T12:32:52,577 INFO [Listener at localhost/35185 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-10-02T12:32:52,577 INFO [Listener at localhost/35185 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-10-02T12:32:52,578 INFO [Listener at localhost/35185 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-10-02T12:32:52,581 INFO [Listener at localhost/35185 {}] http.HttpServer(1219): Jetty bound to port 45253 2023-10-02T12:32:52,581 INFO [Listener at localhost/35185 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-10-02T12:32:52,583 INFO [Listener at localhost/35185 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:32:52,583 INFO [Listener at localhost/35185 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@322f357a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/hadoop.log.dir/,AVAILABLE} 2023-10-02T12:32:52,584 INFO [Listener at localhost/35185 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:32:52,584 INFO [Listener at localhost/35185 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@3bce61c2{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-10-02T12:32:52,596 INFO [Listener at localhost/35185 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-10-02T12:32:52,597 INFO [Listener at localhost/35185 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-10-02T12:32:52,598 INFO [Listener at localhost/35185 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-10-02T12:32:52,598 INFO [Listener at localhost/35185 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2023-10-02T12:32:52,599 INFO [Listener at localhost/35185 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:32:52,604 INFO [Listener at localhost/35185 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@24d48c2{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-10-02T12:32:52,605 INFO [Listener at localhost/35185 {}] server.AbstractConnector(333): Started ServerConnector@3a93225f{HTTP/1.1, (http/1.1)}{0.0.0.0:45253} 2023-10-02T12:32:52,605 INFO [Listener at localhost/35185 {}] server.Server(415): Started @11445ms 2023-10-02T12:32:52,613 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-10-02T12:32:52,619 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@eaa0ccc{HTTP/1.1, (http/1.1)}{0.0.0.0:43057} 2023-10-02T12:32:52,619 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(415): Started @11459ms 2023-10-02T12:32:52,619 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2388): Adding backup master ZNode /1/backup-masters/jenkins-hbase3.apache.org,45341,1696249969200 2023-10-02T12:32:52,632 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-10-02T12:32:52,632 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-10-02T12:32:52,633 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/jenkins-hbase3.apache.org,45341,1696249969200 2023-10-02T12:32:52,644 DEBUG [M:0;jenkins-hbase3:45341 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:45341 2023-10-02T12:32:52,655 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-10-02T12:32:52,655 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-10-02T12:32:52,655 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:32:52,655 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:32:52,657 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Set watcher on existing znode=/1/master 2023-10-02T12:32:52,658 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Set watcher on existing znode=/1/master 2023-10-02T12:32:52,659 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/jenkins-hbase3.apache.org,45341,1696249969200 from backup master directory 2023-10-02T12:32:52,663 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-10-02T12:32:52,663 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/jenkins-hbase3.apache.org,45341,1696249969200 2023-10-02T12:32:52,663 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-10-02T12:32:52,664 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-10-02T12:32:52,665 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,45341,1696249969200 2023-10-02T12:32:52,668 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-10-02T12:32:52,670 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-10-02T12:32:53,166 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(611): Created cluster ID file at hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/hbase.id with ID: 5e0d74f8-d11e-4ed5-b658-16bddfb712df 2023-10-02T12:32:53,210 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:32:53,232 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:32:53,232 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:32:53,321 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:32:53,323 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-10-02T12:32:53,343 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(271): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-10-02T12:32:53,343 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(277): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-10-02T12:32:53,345 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(286): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:284) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:311) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:951) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2413) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:574) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:571) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-10-02T12:32:53,355 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:951) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2413) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:574) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:571) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-10-02T12:32:53,357 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-10-02T12:32:53,432 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/MasterData/data/master/store 2023-10-02T12:32:53,859 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-10-02T12:32:53,860 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:32:53,869 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-10-02T12:32:53,872 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:32:53,874 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:32:53,874 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-10-02T12:32:53,875 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:32:53,875 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:32:53,875 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-10-02T12:32:53,888 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/MasterData/data/master/store/.initializing 2023-10-02T12:32:53,888 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/MasterData/WALs/jenkins-hbase3.apache.org,45341,1696249969200 2023-10-02T12:32:53,900 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-10-02T12:32:53,919 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C45341%2C1696249969200, suffix=, logDir=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/MasterData/WALs/jenkins-hbase3.apache.org,45341,1696249969200, archiveDir=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/MasterData/oldWALs, maxLogs=10 2023-10-02T12:32:53,949 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/MasterData/WALs/jenkins-hbase3.apache.org,45341,1696249969200/jenkins-hbase3.apache.org%2C45341%2C1696249969200.1696249973927, exclude list is [], retry=0 2023-10-02T12:32:53,985 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43099,DS-c62d647f-3988-4c56-a813-dcabca820155,DISK] 2023-10-02T12:32:53,993 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:347) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:431) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:425) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:483) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:478) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:489) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:397) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-10-02T12:32:54,083 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/MasterData/WALs/jenkins-hbase3.apache.org,45341,1696249969200/jenkins-hbase3.apache.org%2C45341%2C1696249969200.1696249973927 2023-10-02T12:32:54,084 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43099,DS-c62d647f-3988-4c56-a813-dcabca820155,DISK]] 2023-10-02T12:32:54,084 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:32:54,085 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:32:54,089 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:32:54,090 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:32:54,159 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:32:54,204 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-10-02T12:32:54,210 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:32:54,213 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:32:54,214 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:32:54,217 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-10-02T12:32:54,217 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:32:54,219 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:32:54,219 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:32:54,222 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-10-02T12:32:54,222 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:32:54,223 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:32:54,223 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:32:54,226 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-10-02T12:32:54,227 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:32:54,228 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:32:54,233 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:32:54,234 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:32:54,253 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-10-02T12:32:54,257 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:32:54,261 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:32:54,262 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10020066240, jitterRate=-0.06680861115455627}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-10-02T12:32:54,269 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-10-02T12:32:54,270 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-10-02T12:32:54,306 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58f583dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:32:54,363 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(860): No meta location available on zookeeper, skip migrating... 2023-10-02T12:32:54,384 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-10-02T12:32:54,385 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(563): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-10-02T12:32:54,388 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-10-02T12:32:54,390 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(583): Recovered RegionProcedureStore lease in 1 msec 2023-10-02T12:32:54,396 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(597): Loaded RegionProcedureStore in 6 msec 2023-10-02T12:32:54,397 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-10-02T12:32:54,432 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-10-02T12:32:54,452 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2023-10-02T12:32:54,455 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/balancer already deleted, retry=false 2023-10-02T12:32:54,458 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-10-02T12:32:54,460 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2023-10-02T12:32:54,461 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/normalizer already deleted, retry=false 2023-10-02T12:32:54,465 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-10-02T12:32:54,473 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2023-10-02T12:32:54,475 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/split already deleted, retry=false 2023-10-02T12:32:54,475 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2023-10-02T12:32:54,478 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/merge already deleted, retry=false 2023-10-02T12:32:54,493 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2023-10-02T12:32:54,495 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/snapshot-cleanup already deleted, retry=false 2023-10-02T12:32:54,500 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-10-02T12:32:54,500 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-10-02T12:32:54,500 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:32:54,500 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:32:54,501 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(804): Active/primary master=jenkins-hbase3.apache.org,45341,1696249969200, sessionid=0x1017329d6f10000, setting cluster-up flag (Was=false) 2023-10-02T12:32:54,510 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-10-02T12:32:54,519 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:32:54,519 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:32:54,524 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2023-10-02T12:32:54,525 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,45341,1696249969200 2023-10-02T12:32:54,530 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:32:54,530 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:32:54,536 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2023-10-02T12:32:54,538 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,45341,1696249969200 2023-10-02T12:32:54,542 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:32:54,542 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:32:54,548 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2023-10-02T12:32:54,549 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,45341,1696249969200 2023-10-02T12:32:54,624 DEBUG [RS:0;jenkins-hbase3:38375 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:38375 2023-10-02T12:32:54,625 INFO [RS:0;jenkins-hbase3:38375 {}] regionserver.HRegionServer(997): ClusterId : 5e0d74f8-d11e-4ed5-b658-16bddfb712df 2023-10-02T12:32:54,627 INFO [RS:0;jenkins-hbase3:38375 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-10-02T12:32:54,630 DEBUG [RS:0;jenkins-hbase3:38375 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-10-02T12:32:54,636 DEBUG [RS:0;jenkins-hbase3:38375 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-10-02T12:32:54,636 DEBUG [RS:0;jenkins-hbase3:38375 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-10-02T12:32:54,640 DEBUG [RS:0;jenkins-hbase3:38375 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-10-02T12:32:54,640 DEBUG [RS:0;jenkins-hbase3:38375 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-10-02T12:32:54,648 DEBUG [RS:0;jenkins-hbase3:38375 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-10-02T12:32:54,650 DEBUG [RS:0;jenkins-hbase3:38375 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cf2b8ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:32:54,652 DEBUG [RS:0;jenkins-hbase3:38375 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@764b4711, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-10-02T12:32:54,660 INFO [RS:0;jenkins-hbase3:38375 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-10-02T12:32:54,660 INFO [RS:0;jenkins-hbase3:38375 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-10-02T12:32:54,660 DEBUG [RS:0;jenkins-hbase3:38375 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-10-02T12:32:54,665 INFO [RS:0;jenkins-hbase3:38375 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,45341,1696249969200 with isa=jenkins-hbase3.apache.org/172.31.12.81:38375, startcode=1696249972520 2023-10-02T12:32:54,694 DEBUG [RS:0;jenkins-hbase3:38375 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-10-02T12:32:54,701 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-10-02T12:32:54,710 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-10-02T12:32:54,716 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-10-02T12:32:54,724 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,45341,1696249969200 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-10-02T12:32:54,730 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-02T12:32:54,731 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-02T12:32:54,731 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-02T12:32:54,731 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-02T12:32:54,731 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-10-02T12:32:54,731 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:32:54,731 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-10-02T12:32:54,731 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:32:54,748 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1696250004748 2023-10-02T12:32:54,751 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-10-02T12:32:54,758 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-10-02T12:32:54,759 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-10-02T12:32:54,759 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-10-02T12:32:54,764 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:32:54,764 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-10-02T12:32:54,769 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-10-02T12:32:54,770 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-10-02T12:32:54,771 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-10-02T12:32:54,772 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-10-02T12:32:54,772 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-10-02T12:32:54,773 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-10-02T12:32:54,775 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-10-02T12:32:54,778 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-10-02T12:32:54,778 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-10-02T12:32:54,781 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:57135, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-10-02T12:32:54,791 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x22751e54 to 127.0.0.1:49524 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:32:54,795 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45341 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3181) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:590) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17393) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-10-02T12:32:54,804 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-10-02T12:32:54,804 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560 2023-10-02T12:32:54,836 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50c966da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:32:54,837 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-10-02T12:32:54,840 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-10-02T12:32:54,841 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-10-02T12:32:54,845 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1696249974845,5,FailOnTimeoutGroup] 2023-10-02T12:32:54,852 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1696249974846,5,FailOnTimeoutGroup] 2023-10-02T12:32:54,853 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-10-02T12:32:54,853 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1635): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-10-02T12:32:54,855 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-10-02T12:32:54,856 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-10-02T12:32:54,863 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:32:54,866 DEBUG [RS:0;jenkins-hbase3:38375 {}] regionserver.HRegionServer(3046): Master is not running yet 2023-10-02T12:32:54,866 WARN [RS:0;jenkins-hbase3:38375 {}] regionserver.HRegionServer(1088): reportForDuty failed; sleeping 100 ms and then retrying. 2023-10-02T12:32:54,867 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-10-02T12:32:54,870 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-10-02T12:32:54,870 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:32:54,872 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:32:54,872 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-10-02T12:32:54,878 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-10-02T12:32:54,879 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:32:54,880 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:32:54,880 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-10-02T12:32:54,883 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-10-02T12:32:54,883 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:32:54,884 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:32:54,886 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/hbase/meta/1588230740 2023-10-02T12:32:54,887 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/hbase/meta/1588230740 2023-10-02T12:32:54,890 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-10-02T12:32:54,894 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-10-02T12:32:54,899 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:32:54,900 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11345141600, jitterRate=0.05659864842891693}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-10-02T12:32:54,904 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-10-02T12:32:54,904 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-10-02T12:32:54,904 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-10-02T12:32:54,904 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-10-02T12:32:54,904 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-10-02T12:32:54,905 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-10-02T12:32:54,907 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-10-02T12:32:54,907 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-10-02T12:32:54,912 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-10-02T12:32:54,912 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-10-02T12:32:54,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-10-02T12:32:54,938 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-10-02T12:32:54,943 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-10-02T12:32:54,968 INFO [RS:0;jenkins-hbase3:38375 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,45341,1696249969200 with isa=jenkins-hbase3.apache.org/172.31.12.81:38375, startcode=1696249972520 2023-10-02T12:32:54,974 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45341 {}] master.ServerManager(403): Registering regionserver=jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:32:54,988 DEBUG [RS:0;jenkins-hbase3:38375 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560 2023-10-02T12:32:54,988 DEBUG [RS:0;jenkins-hbase3:38375 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:45521 2023-10-02T12:32:54,988 DEBUG [RS:0;jenkins-hbase3:38375 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=45493 2023-10-02T12:32:54,997 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-10-02T12:32:54,997 DEBUG [RS:0;jenkins-hbase3:38375 {}] zookeeper.ZKUtil(111): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Set watcher on existing znode=/1/rs/jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:32:54,997 WARN [RS:0;jenkins-hbase3:38375 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-10-02T12:32:54,998 INFO [RS:0;jenkins-hbase3:38375 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-10-02T12:32:54,998 DEBUG [RS:0;jenkins-hbase3:38375 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/WALs/jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:32:55,001 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,38375,1696249972520] 2023-10-02T12:32:55,019 DEBUG [RS:0;jenkins-hbase3:38375 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-10-02T12:32:55,036 INFO [RS:0;jenkins-hbase3:38375 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-10-02T12:32:55,055 INFO [RS:0;jenkins-hbase3:38375 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-10-02T12:32:55,061 INFO [RS:0;jenkins-hbase3:38375 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-10-02T12:32:55,061 INFO [RS:0;jenkins-hbase3:38375 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-10-02T12:32:55,062 INFO [RS:0;jenkins-hbase3:38375 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-10-02T12:32:55,072 INFO [RS:0;jenkins-hbase3:38375 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-10-02T12:32:55,073 DEBUG [RS:0;jenkins-hbase3:38375 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:32:55,073 DEBUG [RS:0;jenkins-hbase3:38375 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:32:55,073 DEBUG [RS:0;jenkins-hbase3:38375 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:32:55,073 DEBUG [RS:0;jenkins-hbase3:38375 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:32:55,073 DEBUG [RS:0;jenkins-hbase3:38375 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:32:55,073 DEBUG [RS:0;jenkins-hbase3:38375 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-10-02T12:32:55,073 DEBUG [RS:0;jenkins-hbase3:38375 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:32:55,073 DEBUG [RS:0;jenkins-hbase3:38375 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:32:55,074 DEBUG [RS:0;jenkins-hbase3:38375 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:32:55,074 DEBUG [RS:0;jenkins-hbase3:38375 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:32:55,074 DEBUG [RS:0;jenkins-hbase3:38375 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:32:55,074 DEBUG [RS:0;jenkins-hbase3:38375 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-10-02T12:32:55,074 DEBUG [RS:0;jenkins-hbase3:38375 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-10-02T12:32:55,075 INFO [RS:0;jenkins-hbase3:38375 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-10-02T12:32:55,075 INFO [RS:0;jenkins-hbase3:38375 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-10-02T12:32:55,075 INFO [RS:0;jenkins-hbase3:38375 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-10-02T12:32:55,076 INFO [RS:0;jenkins-hbase3:38375 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-10-02T12:32:55,076 INFO [RS:0;jenkins-hbase3:38375 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,38375,1696249972520-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-10-02T12:32:55,093 INFO [RS:0;jenkins-hbase3:38375 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-10-02T12:32:55,094 WARN [jenkins-hbase3:45341 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-02T12:32:55,096 INFO [RS:0;jenkins-hbase3:38375 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,38375,1696249972520-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-10-02T12:32:55,112 INFO [RS:0;jenkins-hbase3:38375 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,38375,1696249972520 started 2023-10-02T12:32:55,112 INFO [RS:0;jenkins-hbase3:38375 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,38375,1696249972520, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:38375, sessionid=0x1017329d6f10001 2023-10-02T12:32:55,112 DEBUG [RS:0;jenkins-hbase3:38375 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-10-02T12:32:55,112 DEBUG [RS:0;jenkins-hbase3:38375 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:32:55,113 DEBUG [RS:0;jenkins-hbase3:38375 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,38375,1696249972520' 2023-10-02T12:32:55,113 DEBUG [RS:0;jenkins-hbase3:38375 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2023-10-02T12:32:55,113 DEBUG [RS:0;jenkins-hbase3:38375 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2023-10-02T12:32:55,114 DEBUG [RS:0;jenkins-hbase3:38375 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-10-02T12:32:55,114 DEBUG [RS:0;jenkins-hbase3:38375 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-10-02T12:32:55,114 DEBUG [RS:0;jenkins-hbase3:38375 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,38375,1696249972520' 2023-10-02T12:32:55,114 DEBUG [RS:0;jenkins-hbase3:38375 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:32:55,115 DEBUG [RS:0;jenkins-hbase3:38375 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:32:55,115 INFO [RS:0;jenkins-hbase3:38375 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-10-02T12:32:55,115 DEBUG [RS:0;jenkins-hbase3:38375 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-10-02T12:32:55,115 DEBUG [RS:0;jenkins-hbase3:38375 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-10-02T12:32:55,115 DEBUG [RS:0;jenkins-hbase3:38375 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:32:55,115 DEBUG [RS:0;jenkins-hbase3:38375 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,38375,1696249972520' 2023-10-02T12:32:55,115 DEBUG [RS:0;jenkins-hbase3:38375 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2023-10-02T12:32:55,116 DEBUG [RS:0;jenkins-hbase3:38375 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2023-10-02T12:32:55,116 DEBUG [RS:0;jenkins-hbase3:38375 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-10-02T12:32:55,116 INFO [RS:0;jenkins-hbase3:38375 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-10-02T12:32:55,117 INFO [RS:0;jenkins-hbase3:38375 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-10-02T12:32:55,227 INFO [RS:0;jenkins-hbase3:38375 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-10-02T12:32:55,230 INFO [RS:0;jenkins-hbase3:38375 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C38375%2C1696249972520, suffix=, logDir=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/WALs/jenkins-hbase3.apache.org,38375,1696249972520, archiveDir=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/oldWALs, maxLogs=32 2023-10-02T12:32:55,242 DEBUG [RS:0;jenkins-hbase3:38375 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/WALs/jenkins-hbase3.apache.org,38375,1696249972520/jenkins-hbase3.apache.org%2C38375%2C1696249972520.1696249975232, exclude list is [], retry=0 2023-10-02T12:32:55,249 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43099,DS-c62d647f-3988-4c56-a813-dcabca820155,DISK] 2023-10-02T12:32:55,252 INFO [RS:0;jenkins-hbase3:38375 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/WALs/jenkins-hbase3.apache.org,38375,1696249972520/jenkins-hbase3.apache.org%2C38375%2C1696249972520.1696249975232 2023-10-02T12:32:55,253 DEBUG [RS:0;jenkins-hbase3:38375 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43099,DS-c62d647f-3988-4c56-a813-dcabca820155,DISK]] 2023-10-02T12:32:55,346 DEBUG [jenkins-hbase3:45341 {}] assignment.AssignmentManager(2303): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-10-02T12:32:55,354 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:32:55,361 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,38375,1696249972520, state=OPENING 2023-10-02T12:32:55,370 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-10-02T12:32:55,376 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:32:55,376 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:32:55,377 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-10-02T12:32:55,377 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-10-02T12:32:55,380 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,38375,1696249972520}] 2023-10-02T12:32:55,572 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(721): New admin connection to jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:32:55,575 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2023-10-02T12:32:55,579 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:38070, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-10-02T12:32:55,592 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-10-02T12:32:55,593 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-10-02T12:32:55,594 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-10-02T12:32:55,598 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C38375%2C1696249972520.meta, suffix=.meta, logDir=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/WALs/jenkins-hbase3.apache.org,38375,1696249972520, archiveDir=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/oldWALs, maxLogs=32 2023-10-02T12:32:55,617 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/WALs/jenkins-hbase3.apache.org,38375,1696249972520/jenkins-hbase3.apache.org%2C38375%2C1696249972520.meta.1696249975600.meta, exclude list is [], retry=0 2023-10-02T12:32:55,621 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43099,DS-c62d647f-3988-4c56-a813-dcabca820155,DISK] 2023-10-02T12:32:55,625 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/WALs/jenkins-hbase3.apache.org,38375,1696249972520/jenkins-hbase3.apache.org%2C38375%2C1696249972520.meta.1696249975600.meta 2023-10-02T12:32:55,626 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43099,DS-c62d647f-3988-4c56-a813-dcabca820155,DISK]] 2023-10-02T12:32:55,626 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:32:55,631 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:32:55,632 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-10-02T12:32:55,649 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-10-02T12:32:55,650 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-10-02T12:32:55,655 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-10-02T12:32:55,655 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:32:55,655 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-10-02T12:32:55,655 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-10-02T12:32:55,658 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-10-02T12:32:55,659 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-10-02T12:32:55,660 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:32:55,660 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:32:55,660 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-10-02T12:32:55,662 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-10-02T12:32:55,662 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:32:55,662 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:32:55,663 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-10-02T12:32:55,664 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-10-02T12:32:55,664 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:32:55,665 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:32:55,666 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/hbase/meta/1588230740 2023-10-02T12:32:55,669 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/hbase/meta/1588230740 2023-10-02T12:32:55,672 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-10-02T12:32:55,674 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-10-02T12:32:55,675 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10949286720, jitterRate=0.019731789827346802}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-10-02T12:32:55,676 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-10-02T12:32:55,688 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1696249975563 2023-10-02T12:32:55,708 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:meta,,1.1588230740 2023-10-02T12:32:55,708 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-10-02T12:32:55,709 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:32:55,711 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,38375,1696249972520, state=OPEN 2023-10-02T12:32:55,714 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-10-02T12:32:55,714 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-10-02T12:32:55,714 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-10-02T12:32:55,714 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-10-02T12:32:55,718 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=3, resume processing ppid=2 2023-10-02T12:32:55,718 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,38375,1696249972520 in 334 msec 2023-10-02T12:32:55,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=2, resume processing ppid=1 2023-10-02T12:32:55,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 797 msec 2023-10-02T12:32:55,876 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.2500 sec 2023-10-02T12:32:55,876 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1063): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1696249975876, completionTime=-1 2023-10-02T12:32:55,876 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(830): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-10-02T12:32:55,877 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1629): Joining cluster... 2023-10-02T12:32:55,955 DEBUG [hconnection-0x46f0727e-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:32:55,960 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:38078, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:32:56,001 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1641): Number of RegionServers=1 2023-10-02T12:32:56,001 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1696250036001 2023-10-02T12:32:56,002 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1696250096002 2023-10-02T12:32:56,003 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1648): Joined the cluster in 125 msec 2023-10-02T12:32:56,036 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,45341,1696249969200-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-10-02T12:32:56,036 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,45341,1696249969200-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-10-02T12:32:56,037 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,45341,1696249969200-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-10-02T12:32:56,039 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:45341, period=300000, unit=MILLISECONDS is enabled. 2023-10-02T12:32:56,039 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-10-02T12:32:56,047 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-10-02T12:32:56,061 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-10-02T12:32:56,062 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2368): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-10-02T12:32:56,071 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-10-02T12:32:56,076 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:32:56,079 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:32:56,081 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:32:56,502 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => df86f7708dbdd71bc4105e1e70256952, NAME => 'hbase:namespace,,1696249976062.df86f7708dbdd71bc4105e1e70256952.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560 2023-10-02T12:32:56,513 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1696249976062.df86f7708dbdd71bc4105e1e70256952.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:32:56,513 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing df86f7708dbdd71bc4105e1e70256952, disabling compactions & flushes 2023-10-02T12:32:56,514 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1696249976062.df86f7708dbdd71bc4105e1e70256952. 2023-10-02T12:32:56,514 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1696249976062.df86f7708dbdd71bc4105e1e70256952. 2023-10-02T12:32:56,514 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1696249976062.df86f7708dbdd71bc4105e1e70256952. after waiting 0 ms 2023-10-02T12:32:56,514 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1696249976062.df86f7708dbdd71bc4105e1e70256952. 2023-10-02T12:32:56,514 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1696249976062.df86f7708dbdd71bc4105e1e70256952. 2023-10-02T12:32:56,514 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for df86f7708dbdd71bc4105e1e70256952: 2023-10-02T12:32:56,516 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:32:56,537 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1696249976062.df86f7708dbdd71bc4105e1e70256952.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1696249976519"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249976519"}]},"ts":"1696249976519"} 2023-10-02T12:32:56,583 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:32:56,585 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:32:56,591 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249976585"}]},"ts":"1696249976585"} 2023-10-02T12:32:56,595 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-10-02T12:32:56,599 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=df86f7708dbdd71bc4105e1e70256952, ASSIGN}] 2023-10-02T12:32:56,602 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=df86f7708dbdd71bc4105e1e70256952, ASSIGN 2023-10-02T12:32:56,604 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=df86f7708dbdd71bc4105e1e70256952, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38375,1696249972520; forceNewPlan=false, retain=false 2023-10-02T12:32:56,755 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=df86f7708dbdd71bc4105e1e70256952, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:32:56,759 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure df86f7708dbdd71bc4105e1e70256952, server=jenkins-hbase3.apache.org,38375,1696249972520}] 2023-10-02T12:32:56,919 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1696249976062.df86f7708dbdd71bc4105e1e70256952. 2023-10-02T12:32:56,920 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => df86f7708dbdd71bc4105e1e70256952, NAME => 'hbase:namespace,,1696249976062.df86f7708dbdd71bc4105e1e70256952.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:32:56,921 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:32:56,922 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace df86f7708dbdd71bc4105e1e70256952 2023-10-02T12:32:56,922 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1696249976062.df86f7708dbdd71bc4105e1e70256952.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:32:56,922 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for df86f7708dbdd71bc4105e1e70256952 2023-10-02T12:32:56,922 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for df86f7708dbdd71bc4105e1e70256952 2023-10-02T12:32:56,924 INFO [StoreOpener-df86f7708dbdd71bc4105e1e70256952-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region df86f7708dbdd71bc4105e1e70256952 2023-10-02T12:32:56,926 INFO [StoreOpener-df86f7708dbdd71bc4105e1e70256952-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df86f7708dbdd71bc4105e1e70256952 columnFamilyName info 2023-10-02T12:32:56,926 DEBUG [StoreOpener-df86f7708dbdd71bc4105e1e70256952-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:32:56,927 INFO [StoreOpener-df86f7708dbdd71bc4105e1e70256952-1 {}] regionserver.HStore(324): Store=df86f7708dbdd71bc4105e1e70256952/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:32:56,929 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/hbase/namespace/df86f7708dbdd71bc4105e1e70256952 2023-10-02T12:32:56,929 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/hbase/namespace/df86f7708dbdd71bc4105e1e70256952 2023-10-02T12:32:56,934 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for df86f7708dbdd71bc4105e1e70256952 2023-10-02T12:32:56,938 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/hbase/namespace/df86f7708dbdd71bc4105e1e70256952/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:32:56,939 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened df86f7708dbdd71bc4105e1e70256952; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9397003840, jitterRate=-0.12483581900596619}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:32:56,940 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for df86f7708dbdd71bc4105e1e70256952: 2023-10-02T12:32:56,942 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:namespace,,1696249976062.df86f7708dbdd71bc4105e1e70256952., pid=6, masterSystemTime=1696249976913 2023-10-02T12:32:56,946 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:namespace,,1696249976062.df86f7708dbdd71bc4105e1e70256952. 2023-10-02T12:32:56,946 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1696249976062.df86f7708dbdd71bc4105e1e70256952. 2023-10-02T12:32:56,947 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=df86f7708dbdd71bc4105e1e70256952, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:32:56,955 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=6, resume processing ppid=5 2023-10-02T12:32:56,955 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure df86f7708dbdd71bc4105e1e70256952, server=jenkins-hbase3.apache.org,38375,1696249972520 in 192 msec 2023-10-02T12:32:56,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=5, resume processing ppid=4 2023-10-02T12:32:56,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=df86f7708dbdd71bc4105e1e70256952, ASSIGN in 356 msec 2023-10-02T12:32:56,960 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:32:56,960 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249976960"}]},"ts":"1696249976960"} 2023-10-02T12:32:56,963 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-10-02T12:32:56,967 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:32:56,970 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 904 msec 2023-10-02T12:32:56,976 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Set watcher on znode that does not yet exist, /1/namespace 2023-10-02T12:32:56,978 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/namespace 2023-10-02T12:32:56,978 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:32:56,978 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:32:57,037 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-10-02T12:32:57,069 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:32:57,077 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 48 msec 2023-10-02T12:32:57,089 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-10-02T12:32:57,106 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:32:57,113 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 24 msec 2023-10-02T12:32:57,128 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/default 2023-10-02T12:32:57,132 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/hbase 2023-10-02T12:32:57,132 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1193): Master has completed initialization 4.467sec 2023-10-02T12:32:57,135 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-10-02T12:32:57,137 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-10-02T12:32:57,139 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-10-02T12:32:57,140 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-10-02T12:32:57,141 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-10-02T12:32:57,145 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,45341,1696249969200-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-10-02T12:32:57,148 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,45341,1696249969200-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-10-02T12:32:57,165 DEBUG [Listener at localhost/35185 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2e685c49 to 127.0.0.1:49524 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:32:57,169 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1296): Balancer post startup initialization complete, took 0 seconds 2023-10-02T12:32:57,171 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-10-02T12:32:57,173 DEBUG [Listener at localhost/35185 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57c527a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:32:57,177 DEBUG [Listener at localhost/35185 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-10-02T12:32:57,178 DEBUG [Listener at localhost/35185 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-10-02T12:32:57,192 DEBUG [hconnection-0x61572622-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:32:57,203 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:38090, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:32:57,213 INFO [Listener at localhost/35185 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,45341,1696249969200 2023-10-02T12:32:57,225 INFO [Listener at localhost/35185 {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/hadoop.log.dir so I do NOT create it in target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569 2023-10-02T12:32:57,225 INFO [Listener at localhost/35185 {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/hadoop.tmp.dir so I do NOT create it in target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569 2023-10-02T12:32:57,225 DEBUG [Listener at localhost/35185 {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569 2023-10-02T12:32:57,225 INFO [Listener at localhost/35185 {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/cluster_e48cb9c6-a4ad-e8f6-28ba-f199054ea92b, deleteOnExit=true 2023-10-02T12:32:57,225 INFO [Listener at localhost/35185 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/test.cache.data in system properties and HBase conf 2023-10-02T12:32:57,226 INFO [Listener at localhost/35185 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/hadoop.tmp.dir in system properties and HBase conf 2023-10-02T12:32:57,226 INFO [Listener at localhost/35185 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/hadoop.log.dir in system properties and HBase conf 2023-10-02T12:32:57,226 INFO [Listener at localhost/35185 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/mapreduce.cluster.local.dir in system properties and HBase conf 2023-10-02T12:32:57,226 INFO [Listener at localhost/35185 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-10-02T12:32:57,226 INFO [Listener at localhost/35185 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-10-02T12:32:57,226 DEBUG [Listener at localhost/35185 {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-10-02T12:32:57,226 INFO [Listener at localhost/35185 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-10-02T12:32:57,226 INFO [Listener at localhost/35185 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-10-02T12:32:57,227 INFO [Listener at localhost/35185 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-10-02T12:32:57,227 INFO [Listener at localhost/35185 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-02T12:32:57,227 INFO [Listener at localhost/35185 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-10-02T12:32:57,227 INFO [Listener at localhost/35185 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-10-02T12:32:57,227 INFO [Listener at localhost/35185 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-02T12:32:57,227 INFO [Listener at localhost/35185 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-02T12:32:57,227 INFO [Listener at localhost/35185 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-10-02T12:32:57,227 INFO [Listener at localhost/35185 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/nfs.dump.dir in system properties and HBase conf 2023-10-02T12:32:57,227 INFO [Listener at localhost/35185 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/java.io.tmpdir in system properties and HBase conf 2023-10-02T12:32:57,227 INFO [Listener at localhost/35185 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-02T12:32:57,228 INFO [Listener at localhost/35185 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-10-02T12:32:57,228 INFO [Listener at localhost/35185 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-10-02T12:32:57,241 WARN [Listener at localhost/35185 {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-10-02T12:32:57,241 WARN [Listener at localhost/35185 {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-10-02T12:32:57,283 WARN [Listener at localhost/35185 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:32:57,286 INFO [Listener at localhost/35185 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:32:57,291 INFO [Listener at localhost/35185 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/java.io.tmpdir/Jetty_localhost_41373_hdfs____.ctvauc/webapp 2023-10-02T12:32:57,389 INFO [Listener at localhost/35185 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:41373 2023-10-02T12:32:57,402 WARN [Listener at localhost/35185 {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-10-02T12:32:57,402 WARN [Listener at localhost/35185 {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-10-02T12:32:57,449 WARN [Listener at localhost/46097 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-10-02T12:32:57,470 WARN [Listener at localhost/46097 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-10-02T12:32:57,474 WARN [Listener at localhost/46097 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:32:57,476 INFO [Listener at localhost/46097 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:32:57,485 INFO [Listener at localhost/46097 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/java.io.tmpdir/Jetty_localhost_43549_datanode____zcsfeq/webapp 2023-10-02T12:32:57,586 INFO [Listener at localhost/46097 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:43549 2023-10-02T12:32:57,597 WARN [Listener at localhost/45871 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-10-02T12:32:57,641 WARN [Listener at localhost/45871 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-10-02T12:32:57,643 WARN [Listener at localhost/45871 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:32:57,645 INFO [Listener at localhost/45871 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:32:57,648 INFO [Listener at localhost/45871 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/java.io.tmpdir/Jetty_localhost_38353_datanode____codmcz/webapp 2023-10-02T12:32:57,699 WARN [Thread-202 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-10-02T12:32:57,738 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xfa704e9848074edd: Processing first storage report for DS-40ebc77a-7d35-4d07-aa92-3698075c146e from datanode a0d34491-d14b-4a21-99d8-7739ed2881d7 2023-10-02T12:32:57,739 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xfa704e9848074edd: from storage DS-40ebc77a-7d35-4d07-aa92-3698075c146e node DatanodeRegistration(127.0.0.1:39287, datanodeUuid=a0d34491-d14b-4a21-99d8-7739ed2881d7, infoPort=39835, infoSecurePort=0, ipcPort=45871, storageInfo=lv=-57;cid=testClusterID;nsid=329222539;c=1696249977243), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2023-10-02T12:32:57,739 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xfa704e9848074edd: Processing first storage report for DS-0738b456-3eb7-472d-8fa9-bc25ebdce1b4 from datanode a0d34491-d14b-4a21-99d8-7739ed2881d7 2023-10-02T12:32:57,739 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xfa704e9848074edd: from storage DS-0738b456-3eb7-472d-8fa9-bc25ebdce1b4 node DatanodeRegistration(127.0.0.1:39287, datanodeUuid=a0d34491-d14b-4a21-99d8-7739ed2881d7, infoPort=39835, infoSecurePort=0, ipcPort=45871, storageInfo=lv=-57;cid=testClusterID;nsid=329222539;c=1696249977243), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-10-02T12:32:57,764 INFO [Listener at localhost/45871 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:38353 2023-10-02T12:32:57,779 WARN [Listener at localhost/42613 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-10-02T12:32:57,804 WARN [Listener at localhost/42613 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-10-02T12:32:57,807 WARN [Listener at localhost/42613 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:32:57,808 INFO [Listener at localhost/42613 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:32:57,814 INFO [Listener at localhost/42613 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/java.io.tmpdir/Jetty_localhost_46693_datanode____m9lfcl/webapp 2023-10-02T12:32:57,873 WARN [Thread-237 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-10-02T12:32:57,914 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xc6d69ce00188bf95: Processing first storage report for DS-3e33d570-ddd4-4ed8-89c3-cd422ab1335f from datanode c7b5b48b-45b9-4519-bab6-6809c95a1272 2023-10-02T12:32:57,914 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xc6d69ce00188bf95: from storage DS-3e33d570-ddd4-4ed8-89c3-cd422ab1335f node DatanodeRegistration(127.0.0.1:45335, datanodeUuid=c7b5b48b-45b9-4519-bab6-6809c95a1272, infoPort=40399, infoSecurePort=0, ipcPort=42613, storageInfo=lv=-57;cid=testClusterID;nsid=329222539;c=1696249977243), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2023-10-02T12:32:57,914 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xc6d69ce00188bf95: Processing first storage report for DS-f3e23ae6-dab7-4c95-bc0f-cd91971834f2 from datanode c7b5b48b-45b9-4519-bab6-6809c95a1272 2023-10-02T12:32:57,915 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xc6d69ce00188bf95: from storage DS-f3e23ae6-dab7-4c95-bc0f-cd91971834f2 node DatanodeRegistration(127.0.0.1:45335, datanodeUuid=c7b5b48b-45b9-4519-bab6-6809c95a1272, infoPort=40399, infoSecurePort=0, ipcPort=42613, storageInfo=lv=-57;cid=testClusterID;nsid=329222539;c=1696249977243), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-10-02T12:32:57,928 INFO [Listener at localhost/42613 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:46693 2023-10-02T12:32:57,938 WARN [Listener at localhost/37315 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-10-02T12:32:58,028 WARN [Thread-271 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-10-02T12:32:58,058 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x9acd3e90cce020f3: Processing first storage report for DS-8cae75e2-c66d-4ba3-a6d8-2eb7ee961c73 from datanode 09f18d7a-3acb-4d41-bd6e-50104e61a66d 2023-10-02T12:32:58,058 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x9acd3e90cce020f3: from storage DS-8cae75e2-c66d-4ba3-a6d8-2eb7ee961c73 node DatanodeRegistration(127.0.0.1:41837, datanodeUuid=09f18d7a-3acb-4d41-bd6e-50104e61a66d, infoPort=41375, infoSecurePort=0, ipcPort=37315, storageInfo=lv=-57;cid=testClusterID;nsid=329222539;c=1696249977243), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2023-10-02T12:32:58,059 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x9acd3e90cce020f3: Processing first storage report for DS-7d9b0a12-690a-4cf5-bb7d-0baf8449b925 from datanode 09f18d7a-3acb-4d41-bd6e-50104e61a66d 2023-10-02T12:32:58,059 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x9acd3e90cce020f3: from storage DS-7d9b0a12-690a-4cf5-bb7d-0baf8449b925 node DatanodeRegistration(127.0.0.1:41837, datanodeUuid=09f18d7a-3acb-4d41-bd6e-50104e61a66d, infoPort=41375, infoSecurePort=0, ipcPort=37315, storageInfo=lv=-57;cid=testClusterID;nsid=329222539;c=1696249977243), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2023-10-02T12:32:58,154 DEBUG [Listener at localhost/37315 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569 2023-10-02T12:32:58,154 INFO [Listener at localhost/37315 {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-10-02T12:32:58,154 INFO [Listener at localhost/37315 {}] hbase.HBaseTestingUtility(1143): NOT STARTING DFS 2023-10-02T12:32:58,154 INFO [Listener at localhost/37315 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:32:58,156 INFO [Listener at localhost/37315 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:32:58,183 INFO [Listener at localhost/37315 {}] util.FSUtils(462): Created version file at hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648 with version=8 2023-10-02T12:32:58,183 INFO [Listener at localhost/37315 {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/hbase-staging 2023-10-02T12:32:58,185 DEBUG [Listener at localhost/37315 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-10-02T12:32:58,185 DEBUG [Listener at localhost/37315 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-10-02T12:32:58,185 DEBUG [Listener at localhost/37315 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-10-02T12:32:58,185 DEBUG [Listener at localhost/37315 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-10-02T12:32:58,186 INFO [Listener at localhost/37315 {}] client.ConnectionUtils(128): master/jenkins-hbase3:0 server-side Connection retries=45 2023-10-02T12:32:58,186 INFO [Listener at localhost/37315 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-02T12:32:58,186 INFO [Listener at localhost/37315 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-10-02T12:32:58,186 INFO [Listener at localhost/37315 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-10-02T12:32:58,187 INFO [Listener at localhost/37315 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-02T12:32:58,187 INFO [Listener at localhost/37315 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-10-02T12:32:58,187 INFO [Listener at localhost/37315 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-10-02T12:32:58,187 INFO [Listener at localhost/37315 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-10-02T12:32:58,190 INFO [Listener at localhost/37315 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:33927 2023-10-02T12:32:58,191 INFO [Listener at localhost/37315 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:32:58,192 INFO [Listener at localhost/37315 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:32:58,194 INFO [Listener at localhost/37315 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:33927 connecting to ZooKeeper ensemble=127.0.0.1:49524 2023-10-02T12:32:58,198 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): master:339270x0, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-10-02T12:32:58,200 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:33927-0x1017329d6f10004 connected 2023-10-02T12:32:58,213 DEBUG [Listener at localhost/37315 {}] zookeeper.ZKUtil(113): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Set watcher on znode that does not yet exist, /2/master 2023-10-02T12:32:58,213 DEBUG [Listener at localhost/37315 {}] zookeeper.ZKUtil(113): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Set watcher on znode that does not yet exist, /2/running 2023-10-02T12:32:58,214 DEBUG [Listener at localhost/37315 {}] zookeeper.ZKUtil(113): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Set watcher on znode that does not yet exist, /2/acl 2023-10-02T12:32:58,214 DEBUG [Listener at localhost/37315 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33927 2023-10-02T12:32:58,215 DEBUG [Listener at localhost/37315 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33927 2023-10-02T12:32:58,215 DEBUG [Listener at localhost/37315 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33927 2023-10-02T12:32:58,215 DEBUG [Listener at localhost/37315 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33927 2023-10-02T12:32:58,215 DEBUG [Listener at localhost/37315 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33927 2023-10-02T12:32:58,219 INFO [Listener at localhost/37315 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-10-02T12:32:58,219 INFO [Listener at localhost/37315 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-10-02T12:32:58,219 INFO [Listener at localhost/37315 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-10-02T12:32:58,220 INFO [Listener at localhost/37315 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-10-02T12:32:58,220 INFO [Listener at localhost/37315 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-10-02T12:32:58,220 INFO [Listener at localhost/37315 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-10-02T12:32:58,220 INFO [Listener at localhost/37315 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-10-02T12:32:58,222 INFO [Listener at localhost/37315 {}] http.HttpServer(1219): Jetty bound to port 40183 2023-10-02T12:32:58,222 INFO [Listener at localhost/37315 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-10-02T12:32:58,224 INFO [Listener at localhost/37315 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:32:58,224 INFO [Listener at localhost/37315 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@46194509{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/hadoop.log.dir/,AVAILABLE} 2023-10-02T12:32:58,224 INFO [Listener at localhost/37315 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:32:58,225 INFO [Listener at localhost/37315 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@13d2f11e{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-10-02T12:32:58,237 INFO [Listener at localhost/37315 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-10-02T12:32:58,239 INFO [Listener at localhost/37315 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-10-02T12:32:58,240 INFO [Listener at localhost/37315 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-10-02T12:32:58,240 INFO [Listener at localhost/37315 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2023-10-02T12:32:58,241 INFO [Listener at localhost/37315 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:32:58,332 INFO [Listener at localhost/37315 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@63ff4594{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-10-02T12:32:58,334 INFO [Listener at localhost/37315 {}] server.AbstractConnector(333): Started ServerConnector@37a50ea5{HTTP/1.1, (http/1.1)}{0.0.0.0:40183} 2023-10-02T12:32:58,334 INFO [Listener at localhost/37315 {}] server.Server(415): Started @17173ms 2023-10-02T12:32:58,334 INFO [Listener at localhost/37315 {}] master.HMaster(485): hbase.rootdir=hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648, hbase.cluster.distributed=false 2023-10-02T12:32:58,348 INFO [Listener at localhost/37315 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase3:0 server-side Connection retries=45 2023-10-02T12:32:58,348 INFO [Listener at localhost/37315 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-02T12:32:58,348 INFO [Listener at localhost/37315 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-10-02T12:32:58,348 INFO [Listener at localhost/37315 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-10-02T12:32:58,348 INFO [Listener at localhost/37315 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-02T12:32:58,348 INFO [Listener at localhost/37315 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-10-02T12:32:58,348 INFO [Listener at localhost/37315 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-10-02T12:32:58,349 INFO [Listener at localhost/37315 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-10-02T12:32:58,350 INFO [Listener at localhost/37315 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:35953 2023-10-02T12:32:58,350 INFO [Listener at localhost/37315 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-10-02T12:32:58,352 DEBUG [Listener at localhost/37315 {}] mob.MobFileCache(121): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-10-02T12:32:58,354 INFO [Listener at localhost/37315 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:32:58,356 INFO [Listener at localhost/37315 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:32:58,358 INFO [Listener at localhost/37315 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:35953 connecting to ZooKeeper ensemble=127.0.0.1:49524 2023-10-02T12:32:58,362 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): regionserver:359530x0, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-10-02T12:32:58,364 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:35953-0x1017329d6f10005 connected 2023-10-02T12:32:58,364 DEBUG [Listener at localhost/37315 {}] zookeeper.ZKUtil(113): regionserver:35953-0x1017329d6f10005, quorum=127.0.0.1:49524, baseZNode=/2 Set watcher on znode that does not yet exist, /2/master 2023-10-02T12:32:58,365 DEBUG [Listener at localhost/37315 {}] zookeeper.ZKUtil(113): regionserver:35953-0x1017329d6f10005, quorum=127.0.0.1:49524, baseZNode=/2 Set watcher on znode that does not yet exist, /2/running 2023-10-02T12:32:58,366 DEBUG [Listener at localhost/37315 {}] zookeeper.ZKUtil(113): regionserver:35953-0x1017329d6f10005, quorum=127.0.0.1:49524, baseZNode=/2 Set watcher on znode that does not yet exist, /2/acl 2023-10-02T12:32:58,370 DEBUG [Listener at localhost/37315 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35953 2023-10-02T12:32:58,370 DEBUG [Listener at localhost/37315 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35953 2023-10-02T12:32:58,371 DEBUG [Listener at localhost/37315 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35953 2023-10-02T12:32:58,371 DEBUG [Listener at localhost/37315 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35953 2023-10-02T12:32:58,374 DEBUG [Listener at localhost/37315 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35953 2023-10-02T12:32:58,378 INFO [Listener at localhost/37315 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-10-02T12:32:58,378 INFO [Listener at localhost/37315 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-10-02T12:32:58,378 INFO [Listener at localhost/37315 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-10-02T12:32:58,379 INFO [Listener at localhost/37315 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-10-02T12:32:58,379 INFO [Listener at localhost/37315 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-10-02T12:32:58,379 INFO [Listener at localhost/37315 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-10-02T12:32:58,379 INFO [Listener at localhost/37315 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-10-02T12:32:58,380 INFO [Listener at localhost/37315 {}] http.HttpServer(1219): Jetty bound to port 37793 2023-10-02T12:32:58,380 INFO [Listener at localhost/37315 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-10-02T12:32:58,385 INFO [Listener at localhost/37315 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:32:58,385 INFO [Listener at localhost/37315 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@1c32c707{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/hadoop.log.dir/,AVAILABLE} 2023-10-02T12:32:58,386 INFO [Listener at localhost/37315 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:32:58,386 INFO [Listener at localhost/37315 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@3291cb0b{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-10-02T12:32:58,399 INFO [Listener at localhost/37315 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-10-02T12:32:58,400 INFO [Listener at localhost/37315 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-10-02T12:32:58,400 INFO [Listener at localhost/37315 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-10-02T12:32:58,400 INFO [Listener at localhost/37315 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2023-10-02T12:32:58,402 INFO [Listener at localhost/37315 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:32:58,403 INFO [Listener at localhost/37315 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@74598388{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-10-02T12:32:58,404 INFO [Listener at localhost/37315 {}] server.AbstractConnector(333): Started ServerConnector@3ca329d6{HTTP/1.1, (http/1.1)}{0.0.0.0:37793} 2023-10-02T12:32:58,404 INFO [Listener at localhost/37315 {}] server.Server(415): Started @17243ms 2023-10-02T12:32:58,405 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-10-02T12:32:58,408 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@3a1f71b3{HTTP/1.1, (http/1.1)}{0.0.0.0:38317} 2023-10-02T12:32:58,409 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(415): Started @17248ms 2023-10-02T12:32:58,409 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2388): Adding backup master ZNode /2/backup-masters/jenkins-hbase3.apache.org,33927,1696249978185 2023-10-02T12:32:58,412 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/backup-masters 2023-10-02T12:32:58,412 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35953-0x1017329d6f10005, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/backup-masters 2023-10-02T12:32:58,413 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Set watcher on existing znode=/2/backup-masters/jenkins-hbase3.apache.org,33927,1696249978185 2023-10-02T12:32:58,415 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35953-0x1017329d6f10005, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/2/master 2023-10-02T12:32:58,415 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35953-0x1017329d6f10005, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-10-02T12:32:58,415 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/2/master 2023-10-02T12:32:58,415 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-10-02T12:32:58,416 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Set watcher on existing znode=/2/master 2023-10-02T12:32:58,418 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /2/backup-masters/jenkins-hbase3.apache.org,33927,1696249978185 from backup master directory 2023-10-02T12:32:58,419 DEBUG [M:0;jenkins-hbase3:33927 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:33927 2023-10-02T12:32:58,420 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/2/backup-masters/jenkins-hbase3.apache.org,33927,1696249978185 2023-10-02T12:32:58,420 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35953-0x1017329d6f10005, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/backup-masters 2023-10-02T12:32:58,420 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/backup-masters 2023-10-02T12:32:58,420 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-10-02T12:32:58,420 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Set watcher on existing znode=/2/master 2023-10-02T12:32:58,420 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,33927,1696249978185 2023-10-02T12:32:58,841 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(611): Created cluster ID file at hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/hbase.id with ID: 1035f280-9eae-4adb-b601-626dc42e742d 2023-10-02T12:32:58,859 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:32:58,862 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-10-02T12:32:58,862 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35953-0x1017329d6f10005, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-10-02T12:32:59,278 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:32:59,279 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-10-02T12:32:59,280 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-10-02T12:32:59,699 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/MasterData/data/master/store 2023-10-02T12:33:00,124 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:00,124 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-10-02T12:33:00,124 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:33:00,124 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:33:00,124 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-10-02T12:33:00,124 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:33:00,124 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:33:00,124 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-10-02T12:33:00,126 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/MasterData/data/master/store/.initializing 2023-10-02T12:33:00,126 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:46097/tmp/wal/MasterData/WALs/jenkins-hbase3.apache.org,33927,1696249978185 2023-10-02T12:33:00,128 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-10-02T12:33:00,131 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C33927%2C1696249978185, suffix=, logDir=hdfs://localhost:46097/tmp/wal/MasterData/WALs/jenkins-hbase3.apache.org,33927,1696249978185, archiveDir=hdfs://localhost:46097/tmp/wal/MasterData/oldWALs, maxLogs=10 2023-10-02T12:33:00,148 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /tmp/wal/MasterData/WALs/jenkins-hbase3.apache.org,33927,1696249978185/jenkins-hbase3.apache.org%2C33927%2C1696249978185.1696249980131, exclude list is [], retry=0 2023-10-02T12:33:00,154 DEBUG [RS-EventLoopGroup-6-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41837,DS-8cae75e2-c66d-4ba3-a6d8-2eb7ee961c73,DISK] 2023-10-02T12:33:00,158 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /tmp/wal/MasterData/WALs/jenkins-hbase3.apache.org,33927,1696249978185/jenkins-hbase3.apache.org%2C33927%2C1696249978185.1696249980131 2023-10-02T12:33:00,159 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41837,DS-8cae75e2-c66d-4ba3-a6d8-2eb7ee961c73,DISK]] 2023-10-02T12:33:00,159 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:33:00,159 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:00,159 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:33:00,159 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:33:00,163 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:33:00,165 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-10-02T12:33:00,166 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:00,167 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:33:00,167 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:33:00,170 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-10-02T12:33:00,170 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:00,171 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:33:00,171 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:33:00,173 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-10-02T12:33:00,174 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:00,175 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:33:00,175 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:33:00,177 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-10-02T12:33:00,177 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:00,178 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:33:00,179 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:33:00,180 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:33:00,180 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46097/tmp/wal/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:33:00,184 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-10-02T12:33:00,185 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:33:00,189 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46097/tmp/wal/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:33:00,190 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10281774720, jitterRate=-0.04243510961532593}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-10-02T12:33:00,190 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-10-02T12:33:00,190 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-10-02T12:33:00,196 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4461148e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:33:00,197 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(860): No meta location available on zookeeper, skip migrating... 2023-10-02T12:33:00,198 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-10-02T12:33:00,198 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(563): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-10-02T12:33:00,198 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-10-02T12:33:00,199 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(583): Recovered RegionProcedureStore lease in 0 msec 2023-10-02T12:33:00,200 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(597): Loaded RegionProcedureStore in 0 msec 2023-10-02T12:33:00,200 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-10-02T12:33:00,203 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-10-02T12:33:00,205 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Unable to get data of znode /2/balancer because node does not exist (not necessarily an error) 2023-10-02T12:33:00,206 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /2/balancer already deleted, retry=false 2023-10-02T12:33:00,207 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-10-02T12:33:00,207 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Unable to get data of znode /2/normalizer because node does not exist (not necessarily an error) 2023-10-02T12:33:00,208 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /2/normalizer already deleted, retry=false 2023-10-02T12:33:00,209 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-10-02T12:33:00,210 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Unable to get data of znode /2/switch/split because node does not exist (not necessarily an error) 2023-10-02T12:33:00,213 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /2/switch/split already deleted, retry=false 2023-10-02T12:33:00,214 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Unable to get data of znode /2/switch/merge because node does not exist (not necessarily an error) 2023-10-02T12:33:00,215 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /2/switch/merge already deleted, retry=false 2023-10-02T12:33:00,217 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Unable to get data of znode /2/snapshot-cleanup because node does not exist (not necessarily an error) 2023-10-02T12:33:00,219 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /2/snapshot-cleanup already deleted, retry=false 2023-10-02T12:33:00,220 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/2/running 2023-10-02T12:33:00,220 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35953-0x1017329d6f10005, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/2/running 2023-10-02T12:33:00,220 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-10-02T12:33:00,220 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35953-0x1017329d6f10005, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-10-02T12:33:00,221 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(804): Active/primary master=jenkins-hbase3.apache.org,33927,1696249978185, sessionid=0x1017329d6f10004, setting cluster-up flag (Was=false) 2023-10-02T12:33:00,221 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-10-02T12:33:00,225 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35953-0x1017329d6f10005, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-10-02T12:33:00,225 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-10-02T12:33:00,232 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /2/flush-table-proc/acquired, /2/flush-table-proc/reached, /2/flush-table-proc/abort 2023-10-02T12:33:00,233 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,33927,1696249978185 2023-10-02T12:33:00,239 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35953-0x1017329d6f10005, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-10-02T12:33:00,239 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-10-02T12:33:00,244 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /2/rolllog-proc/acquired, /2/rolllog-proc/reached, /2/rolllog-proc/abort 2023-10-02T12:33:00,245 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,33927,1696249978185 2023-10-02T12:33:00,250 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35953-0x1017329d6f10005, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-10-02T12:33:00,250 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-10-02T12:33:00,257 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /2/online-snapshot/acquired, /2/online-snapshot/reached, /2/online-snapshot/abort 2023-10-02T12:33:00,258 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,33927,1696249978185 2023-10-02T12:33:00,265 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-10-02T12:33:00,265 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-10-02T12:33:00,265 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-10-02T12:33:00,266 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,33927,1696249978185 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-10-02T12:33:00,266 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-02T12:33:00,266 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-02T12:33:00,267 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-02T12:33:00,267 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-02T12:33:00,267 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-10-02T12:33:00,267 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:33:00,267 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-10-02T12:33:00,267 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:33:00,276 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1696250010275 2023-10-02T12:33:00,276 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-10-02T12:33:00,276 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-10-02T12:33:00,276 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-10-02T12:33:00,276 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-10-02T12:33:00,276 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-10-02T12:33:00,276 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-10-02T12:33:00,276 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-10-02T12:33:00,280 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-10-02T12:33:00,280 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-10-02T12:33:00,281 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-10-02T12:33:00,281 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-10-02T12:33:00,281 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-10-02T12:33:00,281 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-10-02T12:33:00,283 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x747ec952 to 127.0.0.1:49524 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:33:00,284 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:00,284 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-10-02T12:33:00,304 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@376344e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:33:00,304 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-10-02T12:33:00,306 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-10-02T12:33:00,306 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-10-02T12:33:00,313 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1696249980310,5,FailOnTimeoutGroup] 2023-10-02T12:33:00,314 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1696249980314,5,FailOnTimeoutGroup] 2023-10-02T12:33:00,314 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-10-02T12:33:00,314 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1635): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-10-02T12:33:00,314 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-10-02T12:33:00,314 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-10-02T12:33:00,341 DEBUG [RS:0;jenkins-hbase3:35953 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:35953 2023-10-02T12:33:00,342 INFO [RS:0;jenkins-hbase3:35953 {}] regionserver.HRegionServer(997): ClusterId : 1035f280-9eae-4adb-b601-626dc42e742d 2023-10-02T12:33:00,342 INFO [RS:0;jenkins-hbase3:35953 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-10-02T12:33:00,342 DEBUG [RS:0;jenkins-hbase3:35953 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-10-02T12:33:00,345 DEBUG [RS:0;jenkins-hbase3:35953 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-10-02T12:33:00,345 DEBUG [RS:0;jenkins-hbase3:35953 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-10-02T12:33:00,348 DEBUG [RS:0;jenkins-hbase3:35953 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-10-02T12:33:00,348 DEBUG [RS:0;jenkins-hbase3:35953 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-10-02T12:33:00,353 DEBUG [RS:0;jenkins-hbase3:35953 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-10-02T12:33:00,355 DEBUG [RS:0;jenkins-hbase3:35953 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d541edc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:33:00,355 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-10-02T12:33:00,356 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-10-02T12:33:00,361 DEBUG [RS:0;jenkins-hbase3:35953 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@426113db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-10-02T12:33:00,361 INFO [RS:0;jenkins-hbase3:35953 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-10-02T12:33:00,361 INFO [RS:0;jenkins-hbase3:35953 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-10-02T12:33:00,361 DEBUG [RS:0;jenkins-hbase3:35953 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-10-02T12:33:00,362 INFO [RS:0;jenkins-hbase3:35953 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,33927,1696249978185 with isa=jenkins-hbase3.apache.org/172.31.12.81:35953, startcode=1696249978347 2023-10-02T12:33:00,363 DEBUG [RS:0;jenkins-hbase3:35953 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-10-02T12:33:00,365 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-10-02T12:33:00,365 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-10-02T12:33:00,367 INFO [RS-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:40235, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2023-10-02T12:33:00,367 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:33:00,367 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2023-10-02T12:33:00,367 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-10-02T12:33:00,368 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-10-02T12:33:00,369 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33927 {}] master.ServerManager(403): Registering regionserver=jenkins-hbase3.apache.org,35953,1696249978347 2023-10-02T12:33:00,369 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-10-02T12:33:00,369 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-10-02T12:33:00,373 DEBUG [RS:0;jenkins-hbase3:35953 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648 2023-10-02T12:33:00,373 DEBUG [RS:0;jenkins-hbase3:35953 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:46097 2023-10-02T12:33:00,373 DEBUG [RS:0;jenkins-hbase3:35953 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=40183 2023-10-02T12:33:00,375 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/rs 2023-10-02T12:33:00,376 DEBUG [RS:0;jenkins-hbase3:35953 {}] zookeeper.ZKUtil(111): regionserver:35953-0x1017329d6f10005, quorum=127.0.0.1:49524, baseZNode=/2 Set watcher on existing znode=/2/rs/jenkins-hbase3.apache.org,35953,1696249978347 2023-10-02T12:33:00,376 WARN [RS:0;jenkins-hbase3:35953 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-10-02T12:33:00,376 INFO [RS:0;jenkins-hbase3:35953 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-10-02T12:33:00,376 DEBUG [RS:0;jenkins-hbase3:35953 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:46097/tmp/wal/WALs/jenkins-hbase3.apache.org,35953,1696249978347 2023-10-02T12:33:00,376 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,35953,1696249978347] 2023-10-02T12:33:00,385 DEBUG [RS:0;jenkins-hbase3:35953 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-10-02T12:33:00,385 INFO [RS:0;jenkins-hbase3:35953 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-10-02T12:33:00,392 INFO [RS:0;jenkins-hbase3:35953 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-10-02T12:33:00,392 INFO [RS:0;jenkins-hbase3:35953 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-10-02T12:33:00,393 INFO [RS:0;jenkins-hbase3:35953 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-10-02T12:33:00,401 INFO [RS:0;jenkins-hbase3:35953 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-10-02T12:33:00,404 INFO [RS:0;jenkins-hbase3:35953 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-10-02T12:33:00,404 DEBUG [RS:0;jenkins-hbase3:35953 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:33:00,404 DEBUG [RS:0;jenkins-hbase3:35953 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:33:00,404 DEBUG [RS:0;jenkins-hbase3:35953 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:33:00,404 DEBUG [RS:0;jenkins-hbase3:35953 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:33:00,404 DEBUG [RS:0;jenkins-hbase3:35953 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:33:00,404 DEBUG [RS:0;jenkins-hbase3:35953 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-10-02T12:33:00,404 DEBUG [RS:0;jenkins-hbase3:35953 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:33:00,404 DEBUG [RS:0;jenkins-hbase3:35953 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:33:00,405 DEBUG [RS:0;jenkins-hbase3:35953 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:33:00,405 DEBUG [RS:0;jenkins-hbase3:35953 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:33:00,405 DEBUG [RS:0;jenkins-hbase3:35953 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:33:00,405 DEBUG [RS:0;jenkins-hbase3:35953 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-10-02T12:33:00,405 DEBUG [RS:0;jenkins-hbase3:35953 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-10-02T12:33:00,407 INFO [RS:0;jenkins-hbase3:35953 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-10-02T12:33:00,407 INFO [RS:0;jenkins-hbase3:35953 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-10-02T12:33:00,407 INFO [RS:0;jenkins-hbase3:35953 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-10-02T12:33:00,407 INFO [RS:0;jenkins-hbase3:35953 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-10-02T12:33:00,407 INFO [RS:0;jenkins-hbase3:35953 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,35953,1696249978347-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-10-02T12:33:00,424 INFO [RS:0;jenkins-hbase3:35953 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-10-02T12:33:00,424 INFO [RS:0;jenkins-hbase3:35953 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,35953,1696249978347-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-10-02T12:33:00,435 INFO [RS:0;jenkins-hbase3:35953 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,35953,1696249978347 started 2023-10-02T12:33:00,435 INFO [RS:0;jenkins-hbase3:35953 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,35953,1696249978347, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:35953, sessionid=0x1017329d6f10005 2023-10-02T12:33:00,435 DEBUG [RS:0;jenkins-hbase3:35953 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-10-02T12:33:00,435 DEBUG [RS:0;jenkins-hbase3:35953 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager jenkins-hbase3.apache.org,35953,1696249978347 2023-10-02T12:33:00,436 DEBUG [RS:0;jenkins-hbase3:35953 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,35953,1696249978347' 2023-10-02T12:33:00,436 DEBUG [RS:0;jenkins-hbase3:35953 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/2/flush-table-proc/abort' 2023-10-02T12:33:00,436 DEBUG [RS:0;jenkins-hbase3:35953 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/2/flush-table-proc/acquired' 2023-10-02T12:33:00,437 DEBUG [RS:0;jenkins-hbase3:35953 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-10-02T12:33:00,437 DEBUG [RS:0;jenkins-hbase3:35953 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-10-02T12:33:00,437 DEBUG [RS:0;jenkins-hbase3:35953 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,35953,1696249978347' 2023-10-02T12:33:00,437 DEBUG [RS:0;jenkins-hbase3:35953 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/2/rolllog-proc/abort' 2023-10-02T12:33:00,437 DEBUG [RS:0;jenkins-hbase3:35953 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/2/rolllog-proc/acquired' 2023-10-02T12:33:00,438 INFO [RS:0;jenkins-hbase3:35953 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-10-02T12:33:00,438 DEBUG [RS:0;jenkins-hbase3:35953 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-10-02T12:33:00,438 DEBUG [RS:0;jenkins-hbase3:35953 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-10-02T12:33:00,438 DEBUG [RS:0;jenkins-hbase3:35953 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,35953,1696249978347 2023-10-02T12:33:00,438 DEBUG [RS:0;jenkins-hbase3:35953 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,35953,1696249978347' 2023-10-02T12:33:00,438 DEBUG [RS:0;jenkins-hbase3:35953 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/2/online-snapshot/abort' 2023-10-02T12:33:00,439 DEBUG [RS:0;jenkins-hbase3:35953 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/2/online-snapshot/acquired' 2023-10-02T12:33:00,439 DEBUG [RS:0;jenkins-hbase3:35953 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-10-02T12:33:00,439 INFO [RS:0;jenkins-hbase3:35953 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-10-02T12:33:00,439 INFO [RS:0;jenkins-hbase3:35953 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-10-02T12:33:00,540 INFO [RS:0;jenkins-hbase3:35953 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-10-02T12:33:00,543 INFO [RS:0;jenkins-hbase3:35953 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C35953%2C1696249978347, suffix=, logDir=hdfs://localhost:46097/tmp/wal/WALs/jenkins-hbase3.apache.org,35953,1696249978347, archiveDir=hdfs://localhost:46097/tmp/wal/oldWALs, maxLogs=32 2023-10-02T12:33:00,555 DEBUG [RS:0;jenkins-hbase3:35953 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /tmp/wal/WALs/jenkins-hbase3.apache.org,35953,1696249978347/jenkins-hbase3.apache.org%2C35953%2C1696249978347.1696249980544, exclude list is [], retry=0 2023-10-02T12:33:00,562 DEBUG [RS-EventLoopGroup-6-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45335,DS-3e33d570-ddd4-4ed8-89c3-cd422ab1335f,DISK] 2023-10-02T12:33:00,567 INFO [RS:0;jenkins-hbase3:35953 {}] wal.AbstractFSWAL(832): New WAL /tmp/wal/WALs/jenkins-hbase3.apache.org,35953,1696249978347/jenkins-hbase3.apache.org%2C35953%2C1696249978347.1696249980544 2023-10-02T12:33:00,567 DEBUG [RS:0;jenkins-hbase3:35953 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:45335,DS-3e33d570-ddd4-4ed8-89c3-cd422ab1335f,DISK]] 2023-10-02T12:33:00,732 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-10-02T12:33:00,733 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648 2023-10-02T12:33:01,151 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:01,153 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-10-02T12:33:01,157 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-10-02T12:33:01,157 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:01,158 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:33:01,158 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-10-02T12:33:01,160 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-10-02T12:33:01,160 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:01,161 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:33:01,161 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-10-02T12:33:01,163 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-10-02T12:33:01,163 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:01,163 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:33:01,165 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/data/hbase/meta/1588230740 2023-10-02T12:33:01,165 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/data/hbase/meta/1588230740 2023-10-02T12:33:01,166 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46097/tmp/wal/data/hbase/meta/1588230740 2023-10-02T12:33:01,168 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-10-02T12:33:01,170 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-10-02T12:33:01,174 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46097/tmp/wal/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:33:01,175 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11153211840, jitterRate=0.03872379660606384}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-10-02T12:33:01,175 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-10-02T12:33:01,175 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-10-02T12:33:01,175 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-10-02T12:33:01,175 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-10-02T12:33:01,175 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-10-02T12:33:01,175 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-10-02T12:33:01,176 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-10-02T12:33:01,176 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-10-02T12:33:01,177 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-10-02T12:33:01,177 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-10-02T12:33:01,178 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-10-02T12:33:01,180 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-10-02T12:33:01,182 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-10-02T12:33:01,202 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2023-10-02T12:33:01,303 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-10-02T12:33:01,304 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-10-02T12:33:01,332 DEBUG [jenkins-hbase3:33927 {}] assignment.AssignmentManager(2303): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-10-02T12:33:01,333 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,35953,1696249978347 2023-10-02T12:33:01,336 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,35953,1696249978347, state=OPENING 2023-10-02T12:33:01,337 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-10-02T12:33:01,339 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-10-02T12:33:01,339 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35953-0x1017329d6f10005, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-10-02T12:33:01,339 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /2/meta-region-server: CHANGED 2023-10-02T12:33:01,340 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,35953,1696249978347}] 2023-10-02T12:33:01,340 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /2/meta-region-server: CHANGED 2023-10-02T12:33:01,494 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(721): New admin connection to jenkins-hbase3.apache.org,35953,1696249978347 2023-10-02T12:33:01,495 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2023-10-02T12:33:01,498 INFO [RS-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41274, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-10-02T12:33:01,503 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-10-02T12:33:01,503 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-10-02T12:33:01,503 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-10-02T12:33:01,506 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C35953%2C1696249978347.meta, suffix=.meta, logDir=hdfs://localhost:46097/tmp/wal/WALs/jenkins-hbase3.apache.org,35953,1696249978347, archiveDir=hdfs://localhost:46097/tmp/wal/oldWALs, maxLogs=32 2023-10-02T12:33:01,518 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /tmp/wal/WALs/jenkins-hbase3.apache.org,35953,1696249978347/jenkins-hbase3.apache.org%2C35953%2C1696249978347.meta.1696249981507.meta, exclude list is [], retry=0 2023-10-02T12:33:01,523 DEBUG [RS-EventLoopGroup-6-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45335,DS-3e33d570-ddd4-4ed8-89c3-cd422ab1335f,DISK] 2023-10-02T12:33:01,526 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /tmp/wal/WALs/jenkins-hbase3.apache.org,35953,1696249978347/jenkins-hbase3.apache.org%2C35953%2C1696249978347.meta.1696249981507.meta 2023-10-02T12:33:01,526 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:45335,DS-3e33d570-ddd4-4ed8-89c3-cd422ab1335f,DISK]] 2023-10-02T12:33:01,526 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:33:01,527 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:33:01,527 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-10-02T12:33:01,527 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-10-02T12:33:01,527 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-10-02T12:33:01,527 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-10-02T12:33:01,527 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:01,527 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-10-02T12:33:01,527 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-10-02T12:33:01,529 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-10-02T12:33:01,530 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-10-02T12:33:01,530 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:01,531 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:33:01,531 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-10-02T12:33:01,532 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-10-02T12:33:01,532 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:01,533 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:33:01,533 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-10-02T12:33:01,534 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-10-02T12:33:01,534 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:01,535 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:33:01,537 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/data/hbase/meta/1588230740 2023-10-02T12:33:01,537 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/data/hbase/meta/1588230740 2023-10-02T12:33:01,539 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46097/tmp/wal/data/hbase/meta/1588230740 2023-10-02T12:33:01,541 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-10-02T12:33:01,543 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-10-02T12:33:01,544 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9655865600, jitterRate=-0.10072743892669678}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-10-02T12:33:01,544 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-10-02T12:33:01,546 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1696249981494 2023-10-02T12:33:01,551 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:meta,,1.1588230740 2023-10-02T12:33:01,551 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-10-02T12:33:01,552 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,35953,1696249978347 2023-10-02T12:33:01,554 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,35953,1696249978347, state=OPEN 2023-10-02T12:33:01,562 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/2/meta-region-server 2023-10-02T12:33:01,562 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35953-0x1017329d6f10005, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/2/meta-region-server 2023-10-02T12:33:01,562 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /2/meta-region-server: CHANGED 2023-10-02T12:33:01,562 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /2/meta-region-server: CHANGED 2023-10-02T12:33:01,565 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=3, resume processing ppid=2 2023-10-02T12:33:01,565 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,35953,1696249978347 in 223 msec 2023-10-02T12:33:01,568 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=2, resume processing ppid=1 2023-10-02T12:33:01,568 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 387 msec 2023-10-02T12:33:01,571 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.3070 sec 2023-10-02T12:33:01,571 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1063): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1696249981571, completionTime=-1 2023-10-02T12:33:01,571 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(830): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-10-02T12:33:01,571 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1629): Joining cluster... 2023-10-02T12:33:01,573 DEBUG [hconnection-0x5e1bcfb6-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:33:01,577 INFO [RS-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41290, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:33:01,579 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1641): Number of RegionServers=1 2023-10-02T12:33:01,579 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1696250041579 2023-10-02T12:33:01,579 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1696250101579 2023-10-02T12:33:01,579 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1648): Joined the cluster in 8 msec 2023-10-02T12:33:01,587 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,33927,1696249978185-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-10-02T12:33:01,587 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,33927,1696249978185-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-10-02T12:33:01,587 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,33927,1696249978185-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-10-02T12:33:01,588 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:33927, period=300000, unit=MILLISECONDS is enabled. 2023-10-02T12:33:01,588 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-10-02T12:33:01,588 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-10-02T12:33:01,588 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2368): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-10-02T12:33:01,589 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-10-02T12:33:01,590 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-10-02T12:33:01,592 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:33:01,592 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:01,593 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:33:02,007 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => c28d048d5108c73bc091dea0057bf483, NAME => 'hbase:namespace,,1696249981588.c28d048d5108c73bc091dea0057bf483.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648 2023-10-02T12:33:02,421 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1696249981588.c28d048d5108c73bc091dea0057bf483.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:02,422 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing c28d048d5108c73bc091dea0057bf483, disabling compactions & flushes 2023-10-02T12:33:02,422 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1696249981588.c28d048d5108c73bc091dea0057bf483. 2023-10-02T12:33:02,422 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1696249981588.c28d048d5108c73bc091dea0057bf483. 2023-10-02T12:33:02,422 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1696249981588.c28d048d5108c73bc091dea0057bf483. after waiting 0 ms 2023-10-02T12:33:02,422 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1696249981588.c28d048d5108c73bc091dea0057bf483. 2023-10-02T12:33:02,422 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1696249981588.c28d048d5108c73bc091dea0057bf483. 2023-10-02T12:33:02,422 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for c28d048d5108c73bc091dea0057bf483: 2023-10-02T12:33:02,423 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:33:02,424 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1696249981588.c28d048d5108c73bc091dea0057bf483.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1696249982424"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249982424"}]},"ts":"1696249982424"} 2023-10-02T12:33:02,428 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:33:02,429 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:33:02,429 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249982429"}]},"ts":"1696249982429"} 2023-10-02T12:33:02,432 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-10-02T12:33:02,437 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=c28d048d5108c73bc091dea0057bf483, ASSIGN}] 2023-10-02T12:33:02,440 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=c28d048d5108c73bc091dea0057bf483, ASSIGN 2023-10-02T12:33:02,441 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=c28d048d5108c73bc091dea0057bf483, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,35953,1696249978347; forceNewPlan=false, retain=false 2023-10-02T12:33:02,592 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=c28d048d5108c73bc091dea0057bf483, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,35953,1696249978347 2023-10-02T12:33:02,596 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure c28d048d5108c73bc091dea0057bf483, server=jenkins-hbase3.apache.org,35953,1696249978347}] 2023-10-02T12:33:02,758 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1696249981588.c28d048d5108c73bc091dea0057bf483. 2023-10-02T12:33:02,758 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => c28d048d5108c73bc091dea0057bf483, NAME => 'hbase:namespace,,1696249981588.c28d048d5108c73bc091dea0057bf483.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:33:02,759 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:33:02,759 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace c28d048d5108c73bc091dea0057bf483 2023-10-02T12:33:02,759 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1696249981588.c28d048d5108c73bc091dea0057bf483.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:02,759 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for c28d048d5108c73bc091dea0057bf483 2023-10-02T12:33:02,759 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for c28d048d5108c73bc091dea0057bf483 2023-10-02T12:33:02,761 INFO [StoreOpener-c28d048d5108c73bc091dea0057bf483-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region c28d048d5108c73bc091dea0057bf483 2023-10-02T12:33:02,764 INFO [StoreOpener-c28d048d5108c73bc091dea0057bf483-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c28d048d5108c73bc091dea0057bf483 columnFamilyName info 2023-10-02T12:33:02,764 DEBUG [StoreOpener-c28d048d5108c73bc091dea0057bf483-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:02,778 INFO [StoreOpener-c28d048d5108c73bc091dea0057bf483-1 {}] regionserver.HStore(324): Store=c28d048d5108c73bc091dea0057bf483/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:33:02,785 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/data/hbase/namespace/c28d048d5108c73bc091dea0057bf483 2023-10-02T12:33:02,786 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/data/hbase/namespace/c28d048d5108c73bc091dea0057bf483 2023-10-02T12:33:02,787 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46097/tmp/wal/data/hbase/namespace/c28d048d5108c73bc091dea0057bf483 2023-10-02T12:33:02,791 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for c28d048d5108c73bc091dea0057bf483 2023-10-02T12:33:02,801 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46097/tmp/wal/data/hbase/namespace/c28d048d5108c73bc091dea0057bf483/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:33:02,803 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened c28d048d5108c73bc091dea0057bf483; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11723847360, jitterRate=0.09186837077140808}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:33:02,803 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for c28d048d5108c73bc091dea0057bf483: 2023-10-02T12:33:02,805 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:namespace,,1696249981588.c28d048d5108c73bc091dea0057bf483., pid=6, masterSystemTime=1696249982749 2023-10-02T12:33:02,809 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:namespace,,1696249981588.c28d048d5108c73bc091dea0057bf483. 2023-10-02T12:33:02,809 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1696249981588.c28d048d5108c73bc091dea0057bf483. 2023-10-02T12:33:02,809 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=c28d048d5108c73bc091dea0057bf483, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,35953,1696249978347 2023-10-02T12:33:02,816 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=6, resume processing ppid=5 2023-10-02T12:33:02,817 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure c28d048d5108c73bc091dea0057bf483, server=jenkins-hbase3.apache.org,35953,1696249978347 in 217 msec 2023-10-02T12:33:02,820 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=5, resume processing ppid=4 2023-10-02T12:33:02,820 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=c28d048d5108c73bc091dea0057bf483, ASSIGN in 379 msec 2023-10-02T12:33:02,822 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:33:02,822 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249982822"}]},"ts":"1696249982822"} 2023-10-02T12:33:02,825 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-10-02T12:33:02,829 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:33:02,832 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 1.2410 sec 2023-10-02T12:33:02,892 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Set watcher on znode that does not yet exist, /2/namespace 2023-10-02T12:33:02,893 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/2/namespace 2023-10-02T12:33:02,893 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35953-0x1017329d6f10005, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-10-02T12:33:02,894 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-10-02T12:33:02,901 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-10-02T12:33:02,915 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/namespace 2023-10-02T12:33:02,920 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 18 msec 2023-10-02T12:33:02,924 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-10-02T12:33:02,934 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/namespace 2023-10-02T12:33:02,938 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 13 msec 2023-10-02T12:33:02,950 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/2/namespace/default 2023-10-02T12:33:02,952 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/2/namespace/hbase 2023-10-02T12:33:02,952 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1193): Master has completed initialization 4.532sec 2023-10-02T12:33:02,952 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-10-02T12:33:02,952 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-10-02T12:33:02,952 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-10-02T12:33:02,952 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-10-02T12:33:02,952 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-10-02T12:33:02,952 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,33927,1696249978185-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-10-02T12:33:02,952 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,33927,1696249978185-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-10-02T12:33:02,955 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1296): Balancer post startup initialization complete, took 0 seconds 2023-10-02T12:33:02,955 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-10-02T12:33:03,015 DEBUG [Listener at localhost/37315 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x042d2e96 to 127.0.0.1:49524 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:33:03,021 DEBUG [Listener at localhost/37315 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@715f4afc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:33:03,023 DEBUG [hconnection-0x72abed8d-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:33:03,025 INFO [RS-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41294, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:33:03,028 INFO [Listener at localhost/37315 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,33927,1696249978185 2023-10-02T12:33:03,028 INFO [Listener at localhost/37315 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-10-02T12:33:03,028 INFO [Listener at localhost/37315 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/test.cache.data in system properties and HBase conf 2023-10-02T12:33:03,028 INFO [Listener at localhost/37315 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/hadoop.tmp.dir in system properties and HBase conf 2023-10-02T12:33:03,028 INFO [Listener at localhost/37315 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/hadoop.log.dir in system properties and HBase conf 2023-10-02T12:33:03,028 INFO [Listener at localhost/37315 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/mapreduce.cluster.local.dir in system properties and HBase conf 2023-10-02T12:33:03,028 INFO [Listener at localhost/37315 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-10-02T12:33:03,028 INFO [Listener at localhost/37315 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-10-02T12:33:03,028 INFO [Listener at localhost/37315 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-10-02T12:33:03,028 INFO [Listener at localhost/37315 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-10-02T12:33:03,028 INFO [Listener at localhost/37315 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-10-02T12:33:03,029 INFO [Listener at localhost/37315 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-02T12:33:03,029 INFO [Listener at localhost/37315 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-10-02T12:33:03,029 INFO [Listener at localhost/37315 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-10-02T12:33:03,029 INFO [Listener at localhost/37315 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-02T12:33:03,029 INFO [Listener at localhost/37315 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-02T12:33:03,029 INFO [Listener at localhost/37315 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-10-02T12:33:03,029 INFO [Listener at localhost/37315 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/nfs.dump.dir in system properties and HBase conf 2023-10-02T12:33:03,029 INFO [Listener at localhost/37315 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/java.io.tmpdir in system properties and HBase conf 2023-10-02T12:33:03,029 INFO [Listener at localhost/37315 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-02T12:33:03,029 INFO [Listener at localhost/37315 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-10-02T12:33:03,029 INFO [Listener at localhost/37315 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-10-02T12:33:04,708 WARN [Thread-466 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:33:05,125 INFO [Thread-466 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:33:05,132 INFO [Thread-466 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_44113_jobhistory____.wmspcz/webapp 2023-10-02T12:33:05,300 INFO [Thread-466 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-10-02T12:33:05,300 INFO [Thread-466 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-10-02T12:33:05,300 INFO [Thread-466 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-10-02T12:33:05,304 INFO [Thread-466 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-10-02T12:33:05,405 INFO [Thread-466 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:33:05,546 INFO [Thread-466 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:33:05,816 INFO [Thread-466 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-10-02T12:33:05,853 INFO [Thread-466 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:44113 2023-10-02T12:33:06,198 WARN [Listener at jenkins-hbase3.apache.org/42685 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:33:06,286 WARN [Listener at jenkins-hbase3.apache.org/42685 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-10-02T12:33:06,287 INFO [Listener at jenkins-hbase3.apache.org/42685 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:33:06,296 INFO [Listener at jenkins-hbase3.apache.org/42685 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_42109_cluster____g3i19j/webapp 2023-10-02T12:33:06,372 INFO [Listener at jenkins-hbase3.apache.org/42685 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-10-02T12:33:06,373 INFO [Listener at jenkins-hbase3.apache.org/42685 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-10-02T12:33:06,373 INFO [Listener at jenkins-hbase3.apache.org/42685 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-10-02T12:33:06,373 INFO [Listener at jenkins-hbase3.apache.org/42685 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-10-02T12:33:06,381 INFO [Listener at jenkins-hbase3.apache.org/42685 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:33:06,588 INFO [Listener at jenkins-hbase3.apache.org/42685 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:33:06,684 INFO [Listener at jenkins-hbase3.apache.org/42685 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:33:06,693 INFO [Listener at jenkins-hbase3.apache.org/42685 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:42109 2023-10-02T12:33:07,043 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:33:07,058 WARN [Listener at jenkins-hbase3.apache.org/44513 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:33:07,096 WARN [Listener at jenkins-hbase3.apache.org/44513 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-10-02T12:33:07,097 INFO [Listener at jenkins-hbase3.apache.org/44513 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:33:07,108 INFO [Listener at jenkins-hbase3.apache.org/44513 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_34597_node____rslhkh/webapp 2023-10-02T12:33:07,182 INFO [Listener at jenkins-hbase3.apache.org/44513 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-10-02T12:33:07,182 INFO [Listener at jenkins-hbase3.apache.org/44513 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-10-02T12:33:07,182 INFO [Listener at jenkins-hbase3.apache.org/44513 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-10-02T12:33:07,183 INFO [Listener at jenkins-hbase3.apache.org/44513 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-10-02T12:33:07,192 INFO [Listener at jenkins-hbase3.apache.org/44513 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:33:07,268 INFO [Listener at jenkins-hbase3.apache.org/44513 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:33:07,368 INFO [Listener at jenkins-hbase3.apache.org/44513 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:33:07,382 INFO [Listener at jenkins-hbase3.apache.org/44513 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:34597 2023-10-02T12:33:07,598 WARN [Listener at jenkins-hbase3.apache.org/46241 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:33:07,604 WARN [Listener at jenkins-hbase3.apache.org/46241 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-10-02T12:33:07,605 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:33:07,612 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_37813_node____jkwm1h/webapp 2023-10-02T12:33:07,666 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-10-02T12:33:07,666 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-10-02T12:33:07,667 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-10-02T12:33:07,667 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-10-02T12:33:07,678 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:33:07,702 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:33:07,796 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:33:07,804 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:37813 2023-10-02T12:33:07,840 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-10-02T12:33:07,840 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] backup.TestBackupBase(301): ROOTDIR hdfs://localhost:45521/backupUT 2023-10-02T12:33:07,851 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] backup.TestBackupBase(306): REMOTE ROOTDIR hdfs://localhost:46097/backupUT 2023-10-02T12:33:07,869 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:33:07,873 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:59506, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:33:07,890 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns1'} 2023-10-02T12:33:07,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] procedure2.ProcedureExecutor(1032): Stored pid=9, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns1 2023-10-02T12:33:07,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-10-02T12:33:07,917 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:33:07,923 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=9, state=SUCCESS; CreateNamespaceProcedure, namespace=ns1 in 30 msec 2023-10-02T12:33:08,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-10-02T12:33:08,018 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns2'} 2023-10-02T12:33:08,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] procedure2.ProcedureExecutor(1032): Stored pid=10, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns2 2023-10-02T12:33:08,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=10 2023-10-02T12:33:08,032 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:33:08,038 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=10, state=SUCCESS; CreateNamespaceProcedure, namespace=ns2 in 18 msec 2023-10-02T12:33:08,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=10 2023-10-02T12:33:08,129 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns3'} 2023-10-02T12:33:08,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] procedure2.ProcedureExecutor(1032): Stored pid=11, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns3 2023-10-02T12:33:08,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=11 2023-10-02T12:33:08,143 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:33:08,148 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=11, state=SUCCESS; CreateNamespaceProcedure, namespace=ns3 in 17 msec 2023-10-02T12:33:08,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=11 2023-10-02T12:33:08,240 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns4'} 2023-10-02T12:33:08,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] procedure2.ProcedureExecutor(1032): Stored pid=12, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns4 2023-10-02T12:33:08,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-10-02T12:33:08,252 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:33:08,265 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=12, state=SUCCESS; CreateNamespaceProcedure, namespace=ns4 in 24 msec 2023-10-02T12:33:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-10-02T12:33:08,363 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'test-1696249987851', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:33:08,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] procedure2.ProcedureExecutor(1032): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=test-1696249987851 2023-10-02T12:33:08,368 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=test-1696249987851 execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:33:08,368 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "test-1696249987851" procId is: 13 2023-10-02T12:33:08,369 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:08,370 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=test-1696249987851 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:33:08,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:33:08,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:33:08,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:33:08,798 INFO [RegionOpenAndInit-test-1696249987851-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 73dd4d167cdf4d422164d1f17ee13a19, NAME => 'test-1696249987851,,1696249988357.73dd4d167cdf4d422164d1f17ee13a19.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1696249987851', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560 2023-10-02T12:33:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:33:09,250 DEBUG [RegionOpenAndInit-test-1696249987851-pool-0 {}] regionserver.HRegion(888): Instantiated test-1696249987851,,1696249988357.73dd4d167cdf4d422164d1f17ee13a19.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:09,250 DEBUG [RegionOpenAndInit-test-1696249987851-pool-0 {}] regionserver.HRegion(1668): Closing 73dd4d167cdf4d422164d1f17ee13a19, disabling compactions & flushes 2023-10-02T12:33:09,250 INFO [RegionOpenAndInit-test-1696249987851-pool-0 {}] regionserver.HRegion(1690): Closing region test-1696249987851,,1696249988357.73dd4d167cdf4d422164d1f17ee13a19. 2023-10-02T12:33:09,250 DEBUG [RegionOpenAndInit-test-1696249987851-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1696249987851,,1696249988357.73dd4d167cdf4d422164d1f17ee13a19. 2023-10-02T12:33:09,250 DEBUG [RegionOpenAndInit-test-1696249987851-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1696249987851,,1696249988357.73dd4d167cdf4d422164d1f17ee13a19. after waiting 0 ms 2023-10-02T12:33:09,250 DEBUG [RegionOpenAndInit-test-1696249987851-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1696249987851,,1696249988357.73dd4d167cdf4d422164d1f17ee13a19. 2023-10-02T12:33:09,250 INFO [RegionOpenAndInit-test-1696249987851-pool-0 {}] regionserver.HRegion(1909): Closed test-1696249987851,,1696249988357.73dd4d167cdf4d422164d1f17ee13a19. 2023-10-02T12:33:09,250 DEBUG [RegionOpenAndInit-test-1696249987851-pool-0 {}] regionserver.HRegion(1622): Region close journal for 73dd4d167cdf4d422164d1f17ee13a19: 2023-10-02T12:33:09,254 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=test-1696249987851 execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:33:09,255 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1696249987851,,1696249988357.73dd4d167cdf4d422164d1f17ee13a19.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1696249989254"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249989254"}]},"ts":"1696249989254"} 2023-10-02T12:33:09,258 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:33:09,259 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=test-1696249987851 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:33:09,260 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1696249987851","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249989259"}]},"ts":"1696249989259"} 2023-10-02T12:33:09,262 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1696249987851, state=ENABLING in hbase:meta 2023-10-02T12:33:09,268 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1696249987851, region=73dd4d167cdf4d422164d1f17ee13a19, ASSIGN}] 2023-10-02T12:33:09,272 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1696249987851, region=73dd4d167cdf4d422164d1f17ee13a19, ASSIGN 2023-10-02T12:33:09,274 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1696249987851, region=73dd4d167cdf4d422164d1f17ee13a19, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38375,1696249972520; forceNewPlan=false, retain=false 2023-10-02T12:33:09,427 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=73dd4d167cdf4d422164d1f17ee13a19, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:09,430 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; OpenRegionProcedure 73dd4d167cdf4d422164d1f17ee13a19, server=jenkins-hbase3.apache.org,38375,1696249972520}] 2023-10-02T12:33:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:33:09,595 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(130): Open test-1696249987851,,1696249988357.73dd4d167cdf4d422164d1f17ee13a19. 2023-10-02T12:33:09,595 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7254): Opening region: {ENCODED => 73dd4d167cdf4d422164d1f17ee13a19, NAME => 'test-1696249987851,,1696249988357.73dd4d167cdf4d422164d1f17ee13a19.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:33:09,595 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:33:09,595 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1696249987851 73dd4d167cdf4d422164d1f17ee13a19 2023-10-02T12:33:09,596 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(888): Instantiated test-1696249987851,,1696249988357.73dd4d167cdf4d422164d1f17ee13a19.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:09,596 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7296): checking encryption for 73dd4d167cdf4d422164d1f17ee13a19 2023-10-02T12:33:09,596 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7299): checking classloading for 73dd4d167cdf4d422164d1f17ee13a19 2023-10-02T12:33:09,597 INFO [StoreOpener-73dd4d167cdf4d422164d1f17ee13a19-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 73dd4d167cdf4d422164d1f17ee13a19 2023-10-02T12:33:09,599 INFO [StoreOpener-73dd4d167cdf4d422164d1f17ee13a19-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73dd4d167cdf4d422164d1f17ee13a19 columnFamilyName f 2023-10-02T12:33:09,599 DEBUG [StoreOpener-73dd4d167cdf4d422164d1f17ee13a19-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:09,600 INFO [StoreOpener-73dd4d167cdf4d422164d1f17ee13a19-1 {}] regionserver.HStore(324): Store=73dd4d167cdf4d422164d1f17ee13a19/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:33:09,601 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/test-1696249987851/73dd4d167cdf4d422164d1f17ee13a19 2023-10-02T12:33:09,602 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/test-1696249987851/73dd4d167cdf4d422164d1f17ee13a19 2023-10-02T12:33:09,605 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1076): writing seq id for 73dd4d167cdf4d422164d1f17ee13a19 2023-10-02T12:33:09,615 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/test-1696249987851/73dd4d167cdf4d422164d1f17ee13a19/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:33:09,616 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): Opened 73dd4d167cdf4d422164d1f17ee13a19; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10447380640, jitterRate=-0.02701185643672943}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:33:09,617 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(992): Region open journal for 73dd4d167cdf4d422164d1f17ee13a19: 2023-10-02T12:33:09,618 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2556): Post open deploy tasks for test-1696249987851,,1696249988357.73dd4d167cdf4d422164d1f17ee13a19., pid=15, masterSystemTime=1696249989583 2023-10-02T12:33:09,621 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2583): Finished post open deploy task for test-1696249987851,,1696249988357.73dd4d167cdf4d422164d1f17ee13a19. 2023-10-02T12:33:09,621 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(158): Opened test-1696249987851,,1696249988357.73dd4d167cdf4d422164d1f17ee13a19. 2023-10-02T12:33:09,622 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=73dd4d167cdf4d422164d1f17ee13a19, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:09,627 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=15, resume processing ppid=14 2023-10-02T12:33:09,627 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=15, ppid=14, state=SUCCESS; OpenRegionProcedure 73dd4d167cdf4d422164d1f17ee13a19, server=jenkins-hbase3.apache.org,38375,1696249972520 in 194 msec 2023-10-02T12:33:09,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=14, resume processing ppid=13 2023-10-02T12:33:09,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=14, ppid=13, state=SUCCESS; TransitRegionStateProcedure table=test-1696249987851, region=73dd4d167cdf4d422164d1f17ee13a19, ASSIGN in 359 msec 2023-10-02T12:33:09,630 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=test-1696249987851 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:33:09,631 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1696249987851","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249989630"}]},"ts":"1696249989630"} 2023-10-02T12:33:09,633 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1696249987851, state=ENABLED in hbase:meta 2023-10-02T12:33:09,635 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=test-1696249987851 execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:33:09,637 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=13, state=SUCCESS; CreateTableProcedure table=test-1696249987851 in 1.2720 sec 2023-10-02T12:33:10,354 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1696249987851 2023-10-02T12:33:10,354 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1696249987851 Metrics about Tables on a single HBase RegionServer 2023-10-02T12:33:10,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:33:10,485 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:test-1696249987851, procId: 13 completed 2023-10-02T12:33:10,487 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x17b23d4a to 127.0.0.1:49524 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:33:10,495 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@603b5329, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:33:10,497 DEBUG [hconnection-0x6fa5ccd6-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:33:10,504 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:52192, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:33:10,517 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] regionserver.HRegion(8210): writing data to region test-1696249987851,,1696249988357.73dd4d167cdf4d422164d1f17ee13a19. with WAL disabled. Data may be lost in the event of a crash. 2023-10-02T12:33:10,628 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns2:test-16962499878511', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:33:10,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] procedure2.ProcedureExecutor(1032): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:test-16962499878511 2023-10-02T12:33:10,633 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16962499878511 execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:33:10,633 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:10,633 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "test-16962499878511" procId is: 16 2023-10-02T12:33:10,635 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:test-16962499878511 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:33:10,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-10-02T12:33:10,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-10-02T12:33:10,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-10-02T12:33:11,052 INFO [RegionOpenAndInit-ns2:test-16962499878511-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 567521894cd008d1b0178b9ce1b98f67, NAME => 'ns2:test-16962499878511,,1696249990628.567521894cd008d1b0178b9ce1b98f67.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16962499878511', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560 2023-10-02T12:33:11,067 DEBUG [RegionOpenAndInit-ns2:test-16962499878511-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16962499878511,,1696249990628.567521894cd008d1b0178b9ce1b98f67.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:11,067 DEBUG [RegionOpenAndInit-ns2:test-16962499878511-pool-0 {}] regionserver.HRegion(1668): Closing 567521894cd008d1b0178b9ce1b98f67, disabling compactions & flushes 2023-10-02T12:33:11,067 INFO [RegionOpenAndInit-ns2:test-16962499878511-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16962499878511,,1696249990628.567521894cd008d1b0178b9ce1b98f67. 2023-10-02T12:33:11,067 DEBUG [RegionOpenAndInit-ns2:test-16962499878511-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16962499878511,,1696249990628.567521894cd008d1b0178b9ce1b98f67. 2023-10-02T12:33:11,067 DEBUG [RegionOpenAndInit-ns2:test-16962499878511-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16962499878511,,1696249990628.567521894cd008d1b0178b9ce1b98f67. after waiting 0 ms 2023-10-02T12:33:11,067 DEBUG [RegionOpenAndInit-ns2:test-16962499878511-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16962499878511,,1696249990628.567521894cd008d1b0178b9ce1b98f67. 2023-10-02T12:33:11,067 INFO [RegionOpenAndInit-ns2:test-16962499878511-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16962499878511,,1696249990628.567521894cd008d1b0178b9ce1b98f67. 2023-10-02T12:33:11,067 DEBUG [RegionOpenAndInit-ns2:test-16962499878511-pool-0 {}] regionserver.HRegion(1622): Region close journal for 567521894cd008d1b0178b9ce1b98f67: 2023-10-02T12:33:11,069 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:test-16962499878511 execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:33:11,069 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16962499878511,,1696249990628.567521894cd008d1b0178b9ce1b98f67.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1696249991069"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249991069"}]},"ts":"1696249991069"} 2023-10-02T12:33:11,071 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:33:11,073 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:test-16962499878511 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:33:11,073 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16962499878511","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249991073"}]},"ts":"1696249991073"} 2023-10-02T12:33:11,075 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16962499878511, state=ENABLING in hbase:meta 2023-10-02T12:33:11,078 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16962499878511, region=567521894cd008d1b0178b9ce1b98f67, ASSIGN}] 2023-10-02T12:33:11,081 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16962499878511, region=567521894cd008d1b0178b9ce1b98f67, ASSIGN 2023-10-02T12:33:11,082 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16962499878511, region=567521894cd008d1b0178b9ce1b98f67, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38375,1696249972520; forceNewPlan=false, retain=false 2023-10-02T12:33:11,235 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=567521894cd008d1b0178b9ce1b98f67, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:11,239 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; OpenRegionProcedure 567521894cd008d1b0178b9ce1b98f67, server=jenkins-hbase3.apache.org,38375,1696249972520}] 2023-10-02T12:33:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-10-02T12:33:11,398 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(130): Open ns2:test-16962499878511,,1696249990628.567521894cd008d1b0178b9ce1b98f67. 2023-10-02T12:33:11,398 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7254): Opening region: {ENCODED => 567521894cd008d1b0178b9ce1b98f67, NAME => 'ns2:test-16962499878511,,1696249990628.567521894cd008d1b0178b9ce1b98f67.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:33:11,399 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:33:11,399 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16962499878511 567521894cd008d1b0178b9ce1b98f67 2023-10-02T12:33:11,399 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(888): Instantiated ns2:test-16962499878511,,1696249990628.567521894cd008d1b0178b9ce1b98f67.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:11,399 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7296): checking encryption for 567521894cd008d1b0178b9ce1b98f67 2023-10-02T12:33:11,400 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7299): checking classloading for 567521894cd008d1b0178b9ce1b98f67 2023-10-02T12:33:11,418 INFO [StoreOpener-567521894cd008d1b0178b9ce1b98f67-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 567521894cd008d1b0178b9ce1b98f67 2023-10-02T12:33:11,420 INFO [StoreOpener-567521894cd008d1b0178b9ce1b98f67-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 567521894cd008d1b0178b9ce1b98f67 columnFamilyName f 2023-10-02T12:33:11,421 DEBUG [StoreOpener-567521894cd008d1b0178b9ce1b98f67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:11,421 INFO [StoreOpener-567521894cd008d1b0178b9ce1b98f67-1 {}] regionserver.HStore(324): Store=567521894cd008d1b0178b9ce1b98f67/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:33:11,423 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/ns2/test-16962499878511/567521894cd008d1b0178b9ce1b98f67 2023-10-02T12:33:11,430 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/ns2/test-16962499878511/567521894cd008d1b0178b9ce1b98f67 2023-10-02T12:33:11,439 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1076): writing seq id for 567521894cd008d1b0178b9ce1b98f67 2023-10-02T12:33:11,443 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/ns2/test-16962499878511/567521894cd008d1b0178b9ce1b98f67/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:33:11,444 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): Opened 567521894cd008d1b0178b9ce1b98f67; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11876243200, jitterRate=0.10606133937835693}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:33:11,446 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(992): Region open journal for 567521894cd008d1b0178b9ce1b98f67: 2023-10-02T12:33:11,447 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:test-16962499878511,,1696249990628.567521894cd008d1b0178b9ce1b98f67., pid=18, masterSystemTime=1696249991392 2023-10-02T12:33:11,452 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:test-16962499878511,,1696249990628.567521894cd008d1b0178b9ce1b98f67. 2023-10-02T12:33:11,452 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(158): Opened ns2:test-16962499878511,,1696249990628.567521894cd008d1b0178b9ce1b98f67. 2023-10-02T12:33:11,452 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=567521894cd008d1b0178b9ce1b98f67, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:11,460 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=18, resume processing ppid=17 2023-10-02T12:33:11,460 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=18, ppid=17, state=SUCCESS; OpenRegionProcedure 567521894cd008d1b0178b9ce1b98f67, server=jenkins-hbase3.apache.org,38375,1696249972520 in 217 msec 2023-10-02T12:33:11,465 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=17, resume processing ppid=16 2023-10-02T12:33:11,465 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=17, ppid=16, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16962499878511, region=567521894cd008d1b0178b9ce1b98f67, ASSIGN in 382 msec 2023-10-02T12:33:11,467 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:test-16962499878511 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:33:11,468 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16962499878511","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249991467"}]},"ts":"1696249991467"} 2023-10-02T12:33:11,470 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16962499878511, state=ENABLED in hbase:meta 2023-10-02T12:33:11,475 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16962499878511 execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:33:11,478 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=16, state=SUCCESS; CreateTableProcedure table=ns2:test-16962499878511 in 846 msec 2023-10-02T12:33:11,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-10-02T12:33:11,744 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns2:test-16962499878511, procId: 16 completed 2023-10-02T12:33:11,750 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] regionserver.HRegion(8210): writing data to region ns2:test-16962499878511,,1696249990628.567521894cd008d1b0178b9ce1b98f67. with WAL disabled. Data may be lost in the event of a crash. 2023-10-02T12:33:11,855 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns3:test-16962499878512', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:33:11,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] procedure2.ProcedureExecutor(1032): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:test-16962499878512 2023-10-02T12:33:11,860 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16962499878512 execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:33:11,860 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:11,861 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "test-16962499878512" procId is: 19 2023-10-02T12:33:11,862 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:test-16962499878512 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:33:11,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-10-02T12:33:11,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-10-02T12:33:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-10-02T12:33:12,289 INFO [RegionOpenAndInit-ns3:test-16962499878512-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 1d3e651667b9d2576aabb8501f40423e, NAME => 'ns3:test-16962499878512,,1696249991854.1d3e651667b9d2576aabb8501f40423e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16962499878512', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560 2023-10-02T12:33:12,316 DEBUG [RegionOpenAndInit-ns3:test-16962499878512-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16962499878512,,1696249991854.1d3e651667b9d2576aabb8501f40423e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:12,316 DEBUG [RegionOpenAndInit-ns3:test-16962499878512-pool-0 {}] regionserver.HRegion(1668): Closing 1d3e651667b9d2576aabb8501f40423e, disabling compactions & flushes 2023-10-02T12:33:12,316 INFO [RegionOpenAndInit-ns3:test-16962499878512-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16962499878512,,1696249991854.1d3e651667b9d2576aabb8501f40423e. 2023-10-02T12:33:12,316 DEBUG [RegionOpenAndInit-ns3:test-16962499878512-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16962499878512,,1696249991854.1d3e651667b9d2576aabb8501f40423e. 2023-10-02T12:33:12,316 DEBUG [RegionOpenAndInit-ns3:test-16962499878512-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16962499878512,,1696249991854.1d3e651667b9d2576aabb8501f40423e. after waiting 0 ms 2023-10-02T12:33:12,316 DEBUG [RegionOpenAndInit-ns3:test-16962499878512-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16962499878512,,1696249991854.1d3e651667b9d2576aabb8501f40423e. 2023-10-02T12:33:12,316 INFO [RegionOpenAndInit-ns3:test-16962499878512-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16962499878512,,1696249991854.1d3e651667b9d2576aabb8501f40423e. 2023-10-02T12:33:12,317 DEBUG [RegionOpenAndInit-ns3:test-16962499878512-pool-0 {}] regionserver.HRegion(1622): Region close journal for 1d3e651667b9d2576aabb8501f40423e: 2023-10-02T12:33:12,318 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:test-16962499878512 execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:33:12,319 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16962499878512,,1696249991854.1d3e651667b9d2576aabb8501f40423e.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1696249992318"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249992318"}]},"ts":"1696249992318"} 2023-10-02T12:33:12,321 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:33:12,322 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:test-16962499878512 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:33:12,323 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16962499878512","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249992323"}]},"ts":"1696249992323"} 2023-10-02T12:33:12,325 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16962499878512, state=ENABLING in hbase:meta 2023-10-02T12:33:12,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16962499878512, region=1d3e651667b9d2576aabb8501f40423e, ASSIGN}] 2023-10-02T12:33:12,334 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16962499878512, region=1d3e651667b9d2576aabb8501f40423e, ASSIGN 2023-10-02T12:33:12,335 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16962499878512, region=1d3e651667b9d2576aabb8501f40423e, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38375,1696249972520; forceNewPlan=false, retain=false 2023-10-02T12:33:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-10-02T12:33:12,491 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=1d3e651667b9d2576aabb8501f40423e, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:12,494 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; OpenRegionProcedure 1d3e651667b9d2576aabb8501f40423e, server=jenkins-hbase3.apache.org,38375,1696249972520}] 2023-10-02T12:33:12,654 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(130): Open ns3:test-16962499878512,,1696249991854.1d3e651667b9d2576aabb8501f40423e. 2023-10-02T12:33:12,655 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7254): Opening region: {ENCODED => 1d3e651667b9d2576aabb8501f40423e, NAME => 'ns3:test-16962499878512,,1696249991854.1d3e651667b9d2576aabb8501f40423e.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:33:12,655 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:33:12,655 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16962499878512 1d3e651667b9d2576aabb8501f40423e 2023-10-02T12:33:12,655 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(888): Instantiated ns3:test-16962499878512,,1696249991854.1d3e651667b9d2576aabb8501f40423e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:12,656 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7296): checking encryption for 1d3e651667b9d2576aabb8501f40423e 2023-10-02T12:33:12,656 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7299): checking classloading for 1d3e651667b9d2576aabb8501f40423e 2023-10-02T12:33:12,658 INFO [StoreOpener-1d3e651667b9d2576aabb8501f40423e-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 1d3e651667b9d2576aabb8501f40423e 2023-10-02T12:33:12,660 INFO [StoreOpener-1d3e651667b9d2576aabb8501f40423e-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d3e651667b9d2576aabb8501f40423e columnFamilyName f 2023-10-02T12:33:12,660 DEBUG [StoreOpener-1d3e651667b9d2576aabb8501f40423e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:12,661 INFO [StoreOpener-1d3e651667b9d2576aabb8501f40423e-1 {}] regionserver.HStore(324): Store=1d3e651667b9d2576aabb8501f40423e/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:33:12,662 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/ns3/test-16962499878512/1d3e651667b9d2576aabb8501f40423e 2023-10-02T12:33:12,663 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/ns3/test-16962499878512/1d3e651667b9d2576aabb8501f40423e 2023-10-02T12:33:12,666 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1076): writing seq id for 1d3e651667b9d2576aabb8501f40423e 2023-10-02T12:33:12,679 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/ns3/test-16962499878512/1d3e651667b9d2576aabb8501f40423e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:33:12,680 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): Opened 1d3e651667b9d2576aabb8501f40423e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10170305600, jitterRate=-0.0528164803981781}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:33:12,681 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(992): Region open journal for 1d3e651667b9d2576aabb8501f40423e: 2023-10-02T12:33:12,682 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:test-16962499878512,,1696249991854.1d3e651667b9d2576aabb8501f40423e., pid=21, masterSystemTime=1696249992649 2023-10-02T12:33:12,685 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:test-16962499878512,,1696249991854.1d3e651667b9d2576aabb8501f40423e. 2023-10-02T12:33:12,685 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(158): Opened ns3:test-16962499878512,,1696249991854.1d3e651667b9d2576aabb8501f40423e. 2023-10-02T12:33:12,686 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=1d3e651667b9d2576aabb8501f40423e, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:12,691 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=21, resume processing ppid=20 2023-10-02T12:33:12,691 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=21, ppid=20, state=SUCCESS; OpenRegionProcedure 1d3e651667b9d2576aabb8501f40423e, server=jenkins-hbase3.apache.org,38375,1696249972520 in 194 msec 2023-10-02T12:33:12,697 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=20, resume processing ppid=19 2023-10-02T12:33:12,697 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=20, ppid=19, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16962499878512, region=1d3e651667b9d2576aabb8501f40423e, ASSIGN in 360 msec 2023-10-02T12:33:12,698 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:test-16962499878512 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:33:12,698 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16962499878512","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249992698"}]},"ts":"1696249992698"} 2023-10-02T12:33:12,700 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16962499878512, state=ENABLED in hbase:meta 2023-10-02T12:33:12,702 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16962499878512 execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:33:12,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=19, state=SUCCESS; CreateTableProcedure table=ns3:test-16962499878512 in 848 msec 2023-10-02T12:33:12,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-10-02T12:33:12,972 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns3:test-16962499878512, procId: 19 completed 2023-10-02T12:33:12,972 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns3:test-16962499878512 get assigned. Timeout = 60000ms 2023-10-02T12:33:12,974 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-02T12:33:12,980 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.HBaseTestingUtility(3581): All regions for table ns3:test-16962499878512 assigned to meta. Checking AM states. 2023-10-02T12:33:12,981 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-02T12:33:12,981 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.HBaseTestingUtility(3601): All regions for table ns3:test-16962499878512 assigned. 2023-10-02T12:33:12,984 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns4:test-16962499878513', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:33:12,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] procedure2.ProcedureExecutor(1032): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns4:test-16962499878513 2023-10-02T12:33:12,988 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16962499878513 execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:33:12,989 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:12,989 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns4" qualifier: "test-16962499878513" procId is: 22 2023-10-02T12:33:12,990 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns4:test-16962499878513 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:33:12,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-10-02T12:33:13,012 INFO [RegionOpenAndInit-ns4:test-16962499878513-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => ced4de67d56921f0b2e4ff571d6ad352, NAME => 'ns4:test-16962499878513,,1696249992983.ced4de67d56921f0b2e4ff571d6ad352.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-16962499878513', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560 2023-10-02T12:33:13,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-10-02T12:33:13,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-10-02T12:33:13,306 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1696249987851' 2023-10-02T12:33:13,307 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-16962499878512' 2023-10-02T12:33:13,308 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-16962499878511' 2023-10-02T12:33:13,426 DEBUG [RegionOpenAndInit-ns4:test-16962499878513-pool-0 {}] regionserver.HRegion(888): Instantiated ns4:test-16962499878513,,1696249992983.ced4de67d56921f0b2e4ff571d6ad352.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:13,426 DEBUG [RegionOpenAndInit-ns4:test-16962499878513-pool-0 {}] regionserver.HRegion(1668): Closing ced4de67d56921f0b2e4ff571d6ad352, disabling compactions & flushes 2023-10-02T12:33:13,427 INFO [RegionOpenAndInit-ns4:test-16962499878513-pool-0 {}] regionserver.HRegion(1690): Closing region ns4:test-16962499878513,,1696249992983.ced4de67d56921f0b2e4ff571d6ad352. 2023-10-02T12:33:13,427 DEBUG [RegionOpenAndInit-ns4:test-16962499878513-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16962499878513,,1696249992983.ced4de67d56921f0b2e4ff571d6ad352. 2023-10-02T12:33:13,427 DEBUG [RegionOpenAndInit-ns4:test-16962499878513-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16962499878513,,1696249992983.ced4de67d56921f0b2e4ff571d6ad352. after waiting 0 ms 2023-10-02T12:33:13,427 DEBUG [RegionOpenAndInit-ns4:test-16962499878513-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16962499878513,,1696249992983.ced4de67d56921f0b2e4ff571d6ad352. 2023-10-02T12:33:13,427 INFO [RegionOpenAndInit-ns4:test-16962499878513-pool-0 {}] regionserver.HRegion(1909): Closed ns4:test-16962499878513,,1696249992983.ced4de67d56921f0b2e4ff571d6ad352. 2023-10-02T12:33:13,427 DEBUG [RegionOpenAndInit-ns4:test-16962499878513-pool-0 {}] regionserver.HRegion(1622): Region close journal for ced4de67d56921f0b2e4ff571d6ad352: 2023-10-02T12:33:13,428 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns4:test-16962499878513 execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:33:13,429 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns4:test-16962499878513,,1696249992983.ced4de67d56921f0b2e4ff571d6ad352.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1696249993429"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249993429"}]},"ts":"1696249993429"} 2023-10-02T12:33:13,432 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:33:13,433 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns4:test-16962499878513 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:33:13,433 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16962499878513","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249993433"}]},"ts":"1696249993433"} 2023-10-02T12:33:13,435 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16962499878513, state=ENABLING in hbase:meta 2023-10-02T12:33:13,439 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16962499878513, region=ced4de67d56921f0b2e4ff571d6ad352, ASSIGN}] 2023-10-02T12:33:13,442 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16962499878513, region=ced4de67d56921f0b2e4ff571d6ad352, ASSIGN 2023-10-02T12:33:13,443 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns4:test-16962499878513, region=ced4de67d56921f0b2e4ff571d6ad352, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38375,1696249972520; forceNewPlan=false, retain=false 2023-10-02T12:33:13,596 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=ced4de67d56921f0b2e4ff571d6ad352, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:13,599 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE; OpenRegionProcedure ced4de67d56921f0b2e4ff571d6ad352, server=jenkins-hbase3.apache.org,38375,1696249972520}] 2023-10-02T12:33:13,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-10-02T12:33:13,757 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(130): Open ns4:test-16962499878513,,1696249992983.ced4de67d56921f0b2e4ff571d6ad352. 2023-10-02T12:33:13,757 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7254): Opening region: {ENCODED => ced4de67d56921f0b2e4ff571d6ad352, NAME => 'ns4:test-16962499878513,,1696249992983.ced4de67d56921f0b2e4ff571d6ad352.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:33:13,757 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:33:13,758 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16962499878513 ced4de67d56921f0b2e4ff571d6ad352 2023-10-02T12:33:13,758 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(888): Instantiated ns4:test-16962499878513,,1696249992983.ced4de67d56921f0b2e4ff571d6ad352.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:13,758 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7296): checking encryption for ced4de67d56921f0b2e4ff571d6ad352 2023-10-02T12:33:13,758 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7299): checking classloading for ced4de67d56921f0b2e4ff571d6ad352 2023-10-02T12:33:13,760 INFO [StoreOpener-ced4de67d56921f0b2e4ff571d6ad352-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region ced4de67d56921f0b2e4ff571d6ad352 2023-10-02T12:33:13,762 INFO [StoreOpener-ced4de67d56921f0b2e4ff571d6ad352-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ced4de67d56921f0b2e4ff571d6ad352 columnFamilyName f 2023-10-02T12:33:13,762 DEBUG [StoreOpener-ced4de67d56921f0b2e4ff571d6ad352-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:13,762 INFO [StoreOpener-ced4de67d56921f0b2e4ff571d6ad352-1 {}] regionserver.HStore(324): Store=ced4de67d56921f0b2e4ff571d6ad352/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:33:13,763 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/ns4/test-16962499878513/ced4de67d56921f0b2e4ff571d6ad352 2023-10-02T12:33:13,764 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/ns4/test-16962499878513/ced4de67d56921f0b2e4ff571d6ad352 2023-10-02T12:33:13,772 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1076): writing seq id for ced4de67d56921f0b2e4ff571d6ad352 2023-10-02T12:33:13,779 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/ns4/test-16962499878513/ced4de67d56921f0b2e4ff571d6ad352/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:33:13,780 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): Opened ced4de67d56921f0b2e4ff571d6ad352; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10548266080, jitterRate=-0.017616167664527893}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:33:13,781 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(992): Region open journal for ced4de67d56921f0b2e4ff571d6ad352: 2023-10-02T12:33:13,782 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2556): Post open deploy tasks for ns4:test-16962499878513,,1696249992983.ced4de67d56921f0b2e4ff571d6ad352., pid=24, masterSystemTime=1696249993752 2023-10-02T12:33:13,785 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2583): Finished post open deploy task for ns4:test-16962499878513,,1696249992983.ced4de67d56921f0b2e4ff571d6ad352. 2023-10-02T12:33:13,785 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(158): Opened ns4:test-16962499878513,,1696249992983.ced4de67d56921f0b2e4ff571d6ad352. 2023-10-02T12:33:13,786 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=ced4de67d56921f0b2e4ff571d6ad352, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:13,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=24, resume processing ppid=23 2023-10-02T12:33:13,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=24, ppid=23, state=SUCCESS; OpenRegionProcedure ced4de67d56921f0b2e4ff571d6ad352, server=jenkins-hbase3.apache.org,38375,1696249972520 in 189 msec 2023-10-02T12:33:13,796 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=23, resume processing ppid=22 2023-10-02T12:33:13,796 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=23, ppid=22, state=SUCCESS; TransitRegionStateProcedure table=ns4:test-16962499878513, region=ced4de67d56921f0b2e4ff571d6ad352, ASSIGN in 354 msec 2023-10-02T12:33:13,797 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns4:test-16962499878513 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:33:13,797 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16962499878513","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249993797"}]},"ts":"1696249993797"} 2023-10-02T12:33:13,799 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16962499878513, state=ENABLED in hbase:meta 2023-10-02T12:33:13,802 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16962499878513 execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:33:13,804 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=22, state=SUCCESS; CreateTableProcedure table=ns4:test-16962499878513 in 818 msec 2023-10-02T12:33:14,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-10-02T12:33:14,102 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns4:test-16962499878513, procId: 22 completed 2023-10-02T12:33:14,102 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns4:test-16962499878513 get assigned. Timeout = 60000ms 2023-10-02T12:33:14,102 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-02T12:33:14,107 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.HBaseTestingUtility(3581): All regions for table ns4:test-16962499878513 assigned to meta. Checking AM states. 2023-10-02T12:33:14,108 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-02T12:33:14,108 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.HBaseTestingUtility(3601): All regions for table ns4:test-16962499878513 assigned. 2023-10-02T12:33:14,109 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x17b23d4a to 127.0.0.1:49524 2023-10-02T12:33:14,109 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:33:14,180 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.ResourceChecker(147): before: backup.TestRemoteRestore#testFullRestoreRemote Thread=815, OpenFileDescriptor=878, MaxFileDescriptor=60000, SystemLoadAverage=471, ProcessCount=181, AvailableMemoryMB=1694 2023-10-02T12:33:14,180 WARN [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.ResourceChecker(130): Thread=815 is superior to 500 2023-10-02T12:33:14,183 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] backup.TestRemoteRestore(70): test remote full backup on a single table 2023-10-02T12:33:14,185 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6c395121 to 127.0.0.1:49524 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:33:14,335 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@572e28b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:33:14,360 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:33:14,362 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:52196, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:33:14,374 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:33:14,376 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42114, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:33:14,387 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'backup'} 2023-10-02T12:33:14,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] procedure2.ProcedureExecutor(1032): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=backup 2023-10-02T12:33:14,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=25 2023-10-02T12:33:14,426 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:33:14,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=25 2023-10-02T12:33:14,545 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=25, state=SUCCESS; CreateNamespaceProcedure, namespace=backup in 156 msec 2023-10-02T12:33:14,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=25 2023-10-02T12:33:14,710 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:33:14,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] procedure2.ProcedureExecutor(1032): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system 2023-10-02T12:33:14,716 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:33:14,716 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:14,716 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2023-10-02T12:33:14,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:33:14,718 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:33:14,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:33:15,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:33:15,105 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:33:15,142 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 7fa9a98238e4b256d46f306c833fe67e, NAME => 'backup:system,,1696249994709.7fa9a98238e4b256d46f306c833fe67e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560 2023-10-02T12:33:15,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:33:15,602 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system,,1696249994709.7fa9a98238e4b256d46f306c833fe67e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:15,603 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1668): Closing 7fa9a98238e4b256d46f306c833fe67e, disabling compactions & flushes 2023-10-02T12:33:15,603 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system,,1696249994709.7fa9a98238e4b256d46f306c833fe67e. 2023-10-02T12:33:15,603 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1696249994709.7fa9a98238e4b256d46f306c833fe67e. 2023-10-02T12:33:15,603 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1696249994709.7fa9a98238e4b256d46f306c833fe67e. after waiting 0 ms 2023-10-02T12:33:15,603 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1696249994709.7fa9a98238e4b256d46f306c833fe67e. 2023-10-02T12:33:15,603 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1909): Closed backup:system,,1696249994709.7fa9a98238e4b256d46f306c833fe67e. 2023-10-02T12:33:15,603 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1622): Region close journal for 7fa9a98238e4b256d46f306c833fe67e: 2023-10-02T12:33:15,605 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:33:15,605 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1696249994709.7fa9a98238e4b256d46f306c833fe67e.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1696249995605"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249995605"}]},"ts":"1696249995605"} 2023-10-02T12:33:15,608 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:33:15,609 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:33:15,609 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249995609"}]},"ts":"1696249995609"} 2023-10-02T12:33:15,611 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-10-02T12:33:15,615 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=7fa9a98238e4b256d46f306c833fe67e, ASSIGN}] 2023-10-02T12:33:15,618 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=7fa9a98238e4b256d46f306c833fe67e, ASSIGN 2023-10-02T12:33:15,619 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=7fa9a98238e4b256d46f306c833fe67e, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38375,1696249972520; forceNewPlan=false, retain=false 2023-10-02T12:33:15,771 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=7fa9a98238e4b256d46f306c833fe67e, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:15,774 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; OpenRegionProcedure 7fa9a98238e4b256d46f306c833fe67e, server=jenkins-hbase3.apache.org,38375,1696249972520}] 2023-10-02T12:33:15,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:33:15,931 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(130): Open backup:system,,1696249994709.7fa9a98238e4b256d46f306c833fe67e. 2023-10-02T12:33:15,931 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7254): Opening region: {ENCODED => 7fa9a98238e4b256d46f306c833fe67e, NAME => 'backup:system,,1696249994709.7fa9a98238e4b256d46f306c833fe67e.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:33:15,932 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:33:15,932 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 7fa9a98238e4b256d46f306c833fe67e 2023-10-02T12:33:15,932 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(888): Instantiated backup:system,,1696249994709.7fa9a98238e4b256d46f306c833fe67e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:15,932 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7296): checking encryption for 7fa9a98238e4b256d46f306c833fe67e 2023-10-02T12:33:15,932 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7299): checking classloading for 7fa9a98238e4b256d46f306c833fe67e 2023-10-02T12:33:15,934 INFO [StoreOpener-7fa9a98238e4b256d46f306c833fe67e-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 7fa9a98238e4b256d46f306c833fe67e 2023-10-02T12:33:15,936 INFO [StoreOpener-7fa9a98238e4b256d46f306c833fe67e-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7fa9a98238e4b256d46f306c833fe67e columnFamilyName meta 2023-10-02T12:33:15,936 DEBUG [StoreOpener-7fa9a98238e4b256d46f306c833fe67e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:15,936 INFO [StoreOpener-7fa9a98238e4b256d46f306c833fe67e-1 {}] regionserver.HStore(324): Store=7fa9a98238e4b256d46f306c833fe67e/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:33:15,936 INFO [StoreOpener-7fa9a98238e4b256d46f306c833fe67e-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 7fa9a98238e4b256d46f306c833fe67e 2023-10-02T12:33:15,938 INFO [StoreOpener-7fa9a98238e4b256d46f306c833fe67e-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7fa9a98238e4b256d46f306c833fe67e columnFamilyName session 2023-10-02T12:33:15,938 DEBUG [StoreOpener-7fa9a98238e4b256d46f306c833fe67e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:15,939 INFO [StoreOpener-7fa9a98238e4b256d46f306c833fe67e-1 {}] regionserver.HStore(324): Store=7fa9a98238e4b256d46f306c833fe67e/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:33:15,940 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e 2023-10-02T12:33:15,940 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e 2023-10-02T12:33:15,942 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-10-02T12:33:15,944 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1076): writing seq id for 7fa9a98238e4b256d46f306c833fe67e 2023-10-02T12:33:15,947 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:33:15,948 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): Opened 7fa9a98238e4b256d46f306c833fe67e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11400381280, jitterRate=0.06174324452877045}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-10-02T12:33:15,948 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(992): Region open journal for 7fa9a98238e4b256d46f306c833fe67e: 2023-10-02T12:33:15,950 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1696249994709.7fa9a98238e4b256d46f306c833fe67e., pid=28, masterSystemTime=1696249995926 2023-10-02T12:33:15,952 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1696249994709.7fa9a98238e4b256d46f306c833fe67e. 2023-10-02T12:33:15,952 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(158): Opened backup:system,,1696249994709.7fa9a98238e4b256d46f306c833fe67e. 2023-10-02T12:33:15,953 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=7fa9a98238e4b256d46f306c833fe67e, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:15,958 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=28, resume processing ppid=27 2023-10-02T12:33:15,958 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=28, ppid=27, state=SUCCESS; OpenRegionProcedure 7fa9a98238e4b256d46f306c833fe67e, server=jenkins-hbase3.apache.org,38375,1696249972520 in 181 msec 2023-10-02T12:33:15,960 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=27, resume processing ppid=26 2023-10-02T12:33:15,960 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=27, ppid=26, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=7fa9a98238e4b256d46f306c833fe67e, ASSIGN in 343 msec 2023-10-02T12:33:15,961 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:33:15,961 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249995961"}]},"ts":"1696249995961"} 2023-10-02T12:33:15,963 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-10-02T12:33:15,965 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:33:15,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=26, state=SUCCESS; CreateTableProcedure table=backup:system in 1.2550 sec 2023-10-02T12:33:16,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:33:16,828 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: backup:system, procId: 26 completed 2023-10-02T12:33:16,849 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:33:16,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] procedure2.ProcedureExecutor(1032): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system_bulk 2023-10-02T12:33:16,856 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:33:16,856 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:16,856 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2023-10-02T12:33:16,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-10-02T12:33:16,862 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:33:16,888 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 737b8e61c96d872ea2ba4221e6c477e8, NAME => 'backup:system_bulk,,1696249996848.737b8e61c96d872ea2ba4221e6c477e8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560 2023-10-02T12:33:16,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-10-02T12:33:17,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-10-02T12:33:17,312 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1696249996848.737b8e61c96d872ea2ba4221e6c477e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:17,313 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1668): Closing 737b8e61c96d872ea2ba4221e6c477e8, disabling compactions & flushes 2023-10-02T12:33:17,313 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1696249996848.737b8e61c96d872ea2ba4221e6c477e8. 2023-10-02T12:33:17,313 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1696249996848.737b8e61c96d872ea2ba4221e6c477e8. 2023-10-02T12:33:17,313 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1696249996848.737b8e61c96d872ea2ba4221e6c477e8. after waiting 0 ms 2023-10-02T12:33:17,313 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1696249996848.737b8e61c96d872ea2ba4221e6c477e8. 2023-10-02T12:33:17,313 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1909): Closed backup:system_bulk,,1696249996848.737b8e61c96d872ea2ba4221e6c477e8. 2023-10-02T12:33:17,313 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1622): Region close journal for 737b8e61c96d872ea2ba4221e6c477e8: 2023-10-02T12:33:17,315 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:33:17,315 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system_bulk,,1696249996848.737b8e61c96d872ea2ba4221e6c477e8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1696249997315"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249997315"}]},"ts":"1696249997315"} 2023-10-02T12:33:17,317 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:33:17,318 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:33:17,319 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249997318"}]},"ts":"1696249997318"} 2023-10-02T12:33:17,322 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2023-10-02T12:33:17,329 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=737b8e61c96d872ea2ba4221e6c477e8, ASSIGN}] 2023-10-02T12:33:17,333 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=737b8e61c96d872ea2ba4221e6c477e8, ASSIGN 2023-10-02T12:33:17,335 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system_bulk, region=737b8e61c96d872ea2ba4221e6c477e8, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38375,1696249972520; forceNewPlan=false, retain=false 2023-10-02T12:33:17,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-10-02T12:33:17,487 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=737b8e61c96d872ea2ba4221e6c477e8, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:17,490 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE; OpenRegionProcedure 737b8e61c96d872ea2ba4221e6c477e8, server=jenkins-hbase3.apache.org,38375,1696249972520}] 2023-10-02T12:33:17,648 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(130): Open backup:system_bulk,,1696249996848.737b8e61c96d872ea2ba4221e6c477e8. 2023-10-02T12:33:17,648 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7254): Opening region: {ENCODED => 737b8e61c96d872ea2ba4221e6c477e8, NAME => 'backup:system_bulk,,1696249996848.737b8e61c96d872ea2ba4221e6c477e8.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:33:17,648 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:33:17,649 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk 737b8e61c96d872ea2ba4221e6c477e8 2023-10-02T12:33:17,649 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1696249996848.737b8e61c96d872ea2ba4221e6c477e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:17,649 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7296): checking encryption for 737b8e61c96d872ea2ba4221e6c477e8 2023-10-02T12:33:17,649 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7299): checking classloading for 737b8e61c96d872ea2ba4221e6c477e8 2023-10-02T12:33:17,651 INFO [StoreOpener-737b8e61c96d872ea2ba4221e6c477e8-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 737b8e61c96d872ea2ba4221e6c477e8 2023-10-02T12:33:17,653 INFO [StoreOpener-737b8e61c96d872ea2ba4221e6c477e8-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 737b8e61c96d872ea2ba4221e6c477e8 columnFamilyName meta 2023-10-02T12:33:17,653 DEBUG [StoreOpener-737b8e61c96d872ea2ba4221e6c477e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:17,654 INFO [StoreOpener-737b8e61c96d872ea2ba4221e6c477e8-1 {}] regionserver.HStore(324): Store=737b8e61c96d872ea2ba4221e6c477e8/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:33:17,654 INFO [StoreOpener-737b8e61c96d872ea2ba4221e6c477e8-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 737b8e61c96d872ea2ba4221e6c477e8 2023-10-02T12:33:17,656 INFO [StoreOpener-737b8e61c96d872ea2ba4221e6c477e8-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 737b8e61c96d872ea2ba4221e6c477e8 columnFamilyName session 2023-10-02T12:33:17,656 DEBUG [StoreOpener-737b8e61c96d872ea2ba4221e6c477e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:17,657 INFO [StoreOpener-737b8e61c96d872ea2ba4221e6c477e8-1 {}] regionserver.HStore(324): Store=737b8e61c96d872ea2ba4221e6c477e8/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:33:17,658 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system_bulk/737b8e61c96d872ea2ba4221e6c477e8 2023-10-02T12:33:17,662 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system_bulk/737b8e61c96d872ea2ba4221e6c477e8 2023-10-02T12:33:17,665 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-10-02T12:33:17,667 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1076): writing seq id for 737b8e61c96d872ea2ba4221e6c477e8 2023-10-02T12:33:17,671 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system_bulk/737b8e61c96d872ea2ba4221e6c477e8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:33:17,672 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): Opened 737b8e61c96d872ea2ba4221e6c477e8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10425058400, jitterRate=-0.029090777039527893}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-10-02T12:33:17,673 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(992): Region open journal for 737b8e61c96d872ea2ba4221e6c477e8: 2023-10-02T12:33:17,674 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system_bulk,,1696249996848.737b8e61c96d872ea2ba4221e6c477e8., pid=31, masterSystemTime=1696249997642 2023-10-02T12:33:17,677 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system_bulk,,1696249996848.737b8e61c96d872ea2ba4221e6c477e8. 2023-10-02T12:33:17,677 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(158): Opened backup:system_bulk,,1696249996848.737b8e61c96d872ea2ba4221e6c477e8. 2023-10-02T12:33:17,678 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=737b8e61c96d872ea2ba4221e6c477e8, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:17,683 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=31, resume processing ppid=30 2023-10-02T12:33:17,683 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=31, ppid=30, state=SUCCESS; OpenRegionProcedure 737b8e61c96d872ea2ba4221e6c477e8, server=jenkins-hbase3.apache.org,38375,1696249972520 in 190 msec 2023-10-02T12:33:17,686 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=30, resume processing ppid=29 2023-10-02T12:33:17,686 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=30, ppid=29, state=SUCCESS; TransitRegionStateProcedure table=backup:system_bulk, region=737b8e61c96d872ea2ba4221e6c477e8, ASSIGN in 354 msec 2023-10-02T12:33:17,687 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:33:17,687 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249997687"}]},"ts":"1696249997687"} 2023-10-02T12:33:17,689 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2023-10-02T12:33:17,693 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:33:17,695 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=29, state=SUCCESS; CreateTableProcedure table=backup:system_bulk in 844 msec 2023-10-02T12:33:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-10-02T12:33:17,966 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: backup:system_bulk, procId: 29 completed 2023-10-02T12:33:17,981 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:33:18,032 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:33:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249998032 (current time:1696249998032). 2023-10-02T12:33:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:33:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:33:18,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1ddcb033 to 127.0.0.1:49524 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:33:18,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19a1df0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:33:18,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:33:18,049 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:52202, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:33:18,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1ddcb033 to 127.0.0.1:49524 2023-10-02T12:33:18,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:33:18,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:33:18,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] procedure2.ProcedureExecutor(1032): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:33:18,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-10-02T12:33:18,069 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:33:18,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:33:18,077 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:33:18,091 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/.hbase-snapshot, current working dir path: hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:33:18,093 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:33:18,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:33:18,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:33:18,509 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:33:18,514 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 7fa9a98238e4b256d46f306c833fe67e}] 2023-10-02T12:33:18,523 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 7fa9a98238e4b256d46f306c833fe67e 2023-10-02T12:33:18,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:33:18,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38375 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2023-10-02T12:33:18,683 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696249994709.7fa9a98238e4b256d46f306c833fe67e. 2023-10-02T12:33:18,687 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2819): Flushing 7fa9a98238e4b256d46f306c833fe67e 2/2 column families, dataSize=45 B heapSize=632 B 2023-10-02T12:33:18,795 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/.tmp/session/cdf06e31c7274ccc94f7501d8ddd6b22 is 49, key is activesession:/session:c/1696249998017/Put/seqid=0 2023-10-02T12:33:18,821 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/.tmp/session/cdf06e31c7274ccc94f7501d8ddd6b22 2023-10-02T12:33:18,898 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/.tmp/session/cdf06e31c7274ccc94f7501d8ddd6b22 as hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/session/cdf06e31c7274ccc94f7501d8ddd6b22 2023-10-02T12:33:18,907 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/session/cdf06e31c7274ccc94f7501d8ddd6b22, entries=1, sequenceid=5, filesize=4.9 K 2023-10-02T12:33:18,912 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3022): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for 7fa9a98238e4b256d46f306c833fe67e in 226ms, sequenceid=5, compaction requested=false 2023-10-02T12:33:18,912 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-10-02T12:33:18,914 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2520): Flush status journal for 7fa9a98238e4b256d46f306c833fe67e: 2023-10-02T12:33:18,914 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696249994709.7fa9a98238e4b256d46f306c833fe67e. for snapshot_backup_system completed. 2023-10-02T12:33:18,915 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696249994709.7fa9a98238e4b256d46f306c833fe67e.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:33:18,922 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:33:18,927 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-10-02T12:33:18,927 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/session/cdf06e31c7274ccc94f7501d8ddd6b22] hfiles 2023-10-02T12:33:18,927 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/session/cdf06e31c7274ccc94f7501d8ddd6b22 for snapshot=snapshot_backup_system 2023-10-02T12:33:19,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:33:19,309 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2023-10-02T12:33:19,310 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-16962499878513' 2023-10-02T12:33:19,346 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696249994709.7fa9a98238e4b256d46f306c833fe67e. 2023-10-02T12:33:19,352 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=33 2023-10-02T12:33:19,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.HMaster(3995): Remote procedure done, pid=33 2023-10-02T12:33:19,356 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 7fa9a98238e4b256d46f306c833fe67e 2023-10-02T12:33:19,358 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 7fa9a98238e4b256d46f306c833fe67e 2023-10-02T12:33:19,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=33, resume processing ppid=32 2023-10-02T12:33:19,362 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:33:19,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=33, ppid=32, state=SUCCESS; SnapshotRegionProcedure 7fa9a98238e4b256d46f306c833fe67e in 847 msec 2023-10-02T12:33:19,364 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:33:19,367 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:33:19,367 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:33:19,370 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:33:19,400 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:33:19,415 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:33:19,416 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:33:19,421 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:33:19,421 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-10-02T12:33:19,423 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=32, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.3600 sec 2023-10-02T12:33:20,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:33:20,179 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: backup:system, procId: 32 completed 2023-10-02T12:33:20,179 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] impl.TableBackupClient(115): Backup backup_1696249994339 started at 1696250000179. 2023-10-02T12:33:20,201 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.TableBackupClient(119): Backup session backup_1696249994339 has been started. 2023-10-02T12:33:20,204 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-10-02T12:33:20,215 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-10-02T12:33:20,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-10-02T12:33:20,224 INFO [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-10-02T12:33:20,225 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-10-02T12:33:20,225 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-10-02T12:33:20,225 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-10-02T12:33:20,226 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-10-02T12:33:20,226 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:33:20,229 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-10-02T12:33:20,229 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:20,229 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:33:20,229 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:33:20,229 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:20,229 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-10-02T12:33:20,230 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-10-02T12:33:20,230 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-10-02T12:33:20,230 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-10-02T12:33:20,230 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:33:20,230 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-10-02T12:33:20,232 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-10-02T12:33:20,232 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-10-02T12:33:20,233 DEBUG [member: 'jenkins-hbase3.apache.org,38375,1696249972520' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-10-02T12:33:20,233 DEBUG [member: 'jenkins-hbase3.apache.org,38375,1696249972520' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-10-02T12:33:20,234 DEBUG [member: 'jenkins-hbase3.apache.org,38375,1696249972520' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-10-02T12:33:20,234 DEBUG [member: 'jenkins-hbase3.apache.org,38375,1696249972520' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-10-02T12:33:20,234 DEBUG [member: 'jenkins-hbase3.apache.org,38375,1696249972520' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,38375,1696249972520' joining acquired barrier for procedure (rolllog) in zk 2023-10-02T12:33:20,236 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:20,236 DEBUG [member: 'jenkins-hbase3.apache.org,38375,1696249972520' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-10-02T12:33:20,236 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:20,236 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:33:20,236 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:33:20,236 DEBUG [member: 'jenkins-hbase3.apache.org,38375,1696249972520' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-10-02T12:33:20,236 DEBUG [member: 'jenkins-hbase3.apache.org,38375,1696249972520' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-10-02T12:33:20,236 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:33:20,237 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:33:20,237 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:33:20,237 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:20,238 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:33:20,238 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,38375,1696249972520' joining acquired barrier for procedure 'rolllog' on coordinator 2023-10-02T12:33:20,238 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-10-02T12:33:20,238 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@55b5146[Count = 0] remaining members to acquire global barrier 2023-10-02T12:33:20,238 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-10-02T12:33:20,239 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:33:20,240 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-10-02T12:33:20,240 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-10-02T12:33:20,240 DEBUG [member: 'jenkins-hbase3.apache.org,38375,1696249972520' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-10-02T12:33:20,240 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:20,240 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-10-02T12:33:20,241 DEBUG [member: 'jenkins-hbase3.apache.org,38375,1696249972520' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-10-02T12:33:20,241 DEBUG [rs(jenkins-hbase3.apache.org,38375,1696249972520)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:20,241 INFO [rs(jenkins-hbase3.apache.org,38375,1696249972520)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1696249975232 highest: 1696249975232 on jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:20,242 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C38375%2C1696249972520:(num 1696249975232) roll requested 2023-10-02T12:33:20,260 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/WALs/jenkins-hbase3.apache.org,38375,1696249972520/jenkins-hbase3.apache.org%2C38375%2C1696249972520.1696250000242, exclude list is [], retry=0 2023-10-02T12:33:20,268 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43099,DS-c62d647f-3988-4c56-a813-dcabca820155,DISK] 2023-10-02T12:33:20,273 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/WALs/jenkins-hbase3.apache.org,38375,1696249972520/jenkins-hbase3.apache.org%2C38375%2C1696249972520.1696249975232 with entries=19, filesize=3.99 KB; new WAL /user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/WALs/jenkins-hbase3.apache.org,38375,1696249972520/jenkins-hbase3.apache.org%2C38375%2C1696249972520.1696250000242 2023-10-02T12:33:20,274 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43099,DS-c62d647f-3988-4c56-a813-dcabca820155,DISK]] 2023-10-02T12:33:20,274 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/WALs/jenkins-hbase3.apache.org,38375,1696249972520/jenkins-hbase3.apache.org%2C38375%2C1696249972520.1696249975232 is not closed yet, will try archiving it next time 2023-10-02T12:33:20,274 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C38375%2C1696249972520.meta:.meta(num 1696249975600) roll requested 2023-10-02T12:33:20,287 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/WALs/jenkins-hbase3.apache.org,38375,1696249972520/jenkins-hbase3.apache.org%2C38375%2C1696249972520.meta.1696250000274.meta, exclude list is [], retry=0 2023-10-02T12:33:20,292 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43099,DS-c62d647f-3988-4c56-a813-dcabca820155,DISK] 2023-10-02T12:33:20,294 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/WALs/jenkins-hbase3.apache.org,38375,1696249972520/jenkins-hbase3.apache.org%2C38375%2C1696249972520.meta.1696249975600.meta with entries=36, filesize=11.13 KB; new WAL /user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/WALs/jenkins-hbase3.apache.org,38375,1696249972520/jenkins-hbase3.apache.org%2C38375%2C1696249972520.meta.1696250000274.meta 2023-10-02T12:33:20,294 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43099,DS-c62d647f-3988-4c56-a813-dcabca820155,DISK]] 2023-10-02T12:33:20,294 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/WALs/jenkins-hbase3.apache.org,38375,1696249972520/jenkins-hbase3.apache.org%2C38375%2C1696249972520.meta.1696249975600.meta is not closed yet, will try archiving it next time 2023-10-02T12:33:20,302 DEBUG [rs(jenkins-hbase3.apache.org,38375,1696249972520)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2023-10-02T12:33:20,302 INFO [rs(jenkins-hbase3.apache.org,38375,1696249972520)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1696250000242 on jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:20,302 DEBUG [rs(jenkins-hbase3.apache.org,38375,1696249972520)-backup-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:33:20,305 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48362, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-10-02T12:33:20,323 DEBUG [member: 'jenkins-hbase3.apache.org,38375,1696249972520' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-10-02T12:33:20,323 DEBUG [member: 'jenkins-hbase3.apache.org,38375,1696249972520' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,38375,1696249972520' in zk 2023-10-02T12:33:20,325 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:20,325 DEBUG [member: 'jenkins-hbase3.apache.org,38375,1696249972520' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-10-02T12:33:20,325 DEBUG [member: 'jenkins-hbase3.apache.org,38375,1696249972520' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:33:20,325 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:20,325 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:33:20,325 DEBUG [member: 'jenkins-hbase3.apache.org,38375,1696249972520' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-10-02T12:33:20,326 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:33:20,327 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:33:20,327 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:33:20,327 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:33:20,328 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:20,328 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:33:20,328 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:33:20,329 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:20,329 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,38375,1696249972520': 2023-10-02T12:33:20,329 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,38375,1696249972520' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-10-02T12:33:20,329 INFO [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-10-02T12:33:20,330 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-10-02T12:33:20,330 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-10-02T12:33:20,330 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-10-02T12:33:20,330 INFO [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-10-02T12:33:20,333 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:33:20,333 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:33:20,333 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-10-02T12:33:20,333 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-10-02T12:33:20,333 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-10-02T12:33:20,333 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:33:20,333 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:33:20,333 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:33:20,334 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:33:20,334 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:33:20,334 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:33:20,334 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:20,334 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:33:20,334 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:33:20,335 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:33:20,335 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:33:20,335 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:20,336 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:20,336 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:33:20,336 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:33:20,337 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:20,343 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:20,343 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-10-02T12:33:20,343 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-10-02T12:33:20,343 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-10-02T12:33:20,343 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:20,343 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:33:20,343 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-10-02T12:33:20,343 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:33:20,343 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:33:20,343 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:33:20,343 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:33:20,343 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:33:20,343 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:33:20,343 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-10-02T12:33:20,345 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:33:20,345 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-10-02T12:33:20,347 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] client.HBaseAdmin(2838): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-10-02T12:33:20,347 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] client.HBaseAdmin(2846): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-10-02T12:33:20,354 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-10-02T12:33:20,354 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-10-02T12:33:20,355 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-10-02T12:33:20,355 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2023-10-02T12:33:20,355 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16962499878513 2023-10-02T12:33:20,355 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-16962499878513 Metrics about Tables on a single HBase RegionServer 2023-10-02T12:33:20,356 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16962499878512 2023-10-02T12:33:20,356 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-16962499878512 Metrics about Tables on a single HBase RegionServer 2023-10-02T12:33:20,356 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16962499878511 2023-10-02T12:33:20,356 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-16962499878511 Metrics about Tables on a single HBase RegionServer 2023-10-02T12:33:20,447 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] client.HBaseAdmin(2852): Getting current status of procedure from master... 2023-10-02T12:33:20,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-10-02T12:33:20,458 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696250000457_default_test-1696249987851 table=test-1696249987851 type=FLUSH ttl=0 } 2023-10-02T12:33:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250000458 (current time:1696250000458). 2023-10-02T12:33:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:33:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696250000457_default_test-1696249987851 VERSION not specified, setting to 2 2023-10-02T12:33:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x03347cf0 to 127.0.0.1:49524 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:33:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b39597f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:33:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:33:20,471 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46912, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:33:20,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x03347cf0 to 127.0.0.1:49524 2023-10-02T12:33:20,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:33:20,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:33:20,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] procedure2.ProcedureExecutor(1032): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250000457_default_test-1696249987851 table=test-1696249987851 type=FLUSH ttl=0 } 2023-10-02T12:33:20,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696250000457_default_test-1696249987851 table=test-1696249987851 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-10-02T12:33:20,477 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250000457_default_test-1696249987851 table=test-1696249987851 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:33:20,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:33:20,480 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250000457_default_test-1696249987851 table=test-1696249987851 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:33:20,484 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/.hbase-snapshot, current working dir path: hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/.hbase-snapshot/.tmp/snapshot_1696250000457_default_test-1696249987851 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:33:20,486 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250000457_default_test-1696249987851 table=test-1696249987851 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:33:20,508 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250000457_default_test-1696249987851 table=test-1696249987851 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:33:20,508 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 73dd4d167cdf4d422164d1f17ee13a19}] 2023-10-02T12:33:20,511 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 73dd4d167cdf4d422164d1f17ee13a19 2023-10-02T12:33:20,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:33:20,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38375 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2023-10-02T12:33:20,663 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1696249987851,,1696249988357.73dd4d167cdf4d422164d1f17ee13a19. 2023-10-02T12:33:20,665 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2819): Flushing 73dd4d167cdf4d422164d1f17ee13a19 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-10-02T12:33:20,687 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/test-1696249987851/73dd4d167cdf4d422164d1f17ee13a19/.tmp/f/8563d73b96cc4f0a84dda30b6f1539b2 is 37, key is row10/f:q1/1696249990530/Put/seqid=0 2023-10-02T12:33:20,700 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/test-1696249987851/73dd4d167cdf4d422164d1f17ee13a19/.tmp/f/8563d73b96cc4f0a84dda30b6f1539b2 2023-10-02T12:33:20,712 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/test-1696249987851/73dd4d167cdf4d422164d1f17ee13a19/.tmp/f/8563d73b96cc4f0a84dda30b6f1539b2 as hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/test-1696249987851/73dd4d167cdf4d422164d1f17ee13a19/f/8563d73b96cc4f0a84dda30b6f1539b2 2023-10-02T12:33:20,720 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/test-1696249987851/73dd4d167cdf4d422164d1f17ee13a19/f/8563d73b96cc4f0a84dda30b6f1539b2, entries=99, sequenceid=103, filesize=8.2 K 2023-10-02T12:33:20,723 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 73dd4d167cdf4d422164d1f17ee13a19 in 58ms, sequenceid=103, compaction requested=false 2023-10-02T12:33:20,724 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2520): Flush status journal for 73dd4d167cdf4d422164d1f17ee13a19: 2023-10-02T12:33:20,724 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1696249987851,,1696249988357.73dd4d167cdf4d422164d1f17ee13a19. for snapshot_1696250000457_default_test-1696249987851 completed. 2023-10-02T12:33:20,725 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(240): Storing 'test-1696249987851,,1696249988357.73dd4d167cdf4d422164d1f17ee13a19.' region-info for snapshot=snapshot_1696250000457_default_test-1696249987851 2023-10-02T12:33:20,725 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:33:20,725 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/test-1696249987851/73dd4d167cdf4d422164d1f17ee13a19/f/8563d73b96cc4f0a84dda30b6f1539b2] hfiles 2023-10-02T12:33:20,725 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/test-1696249987851/73dd4d167cdf4d422164d1f17ee13a19/f/8563d73b96cc4f0a84dda30b6f1539b2 for snapshot=snapshot_1696250000457_default_test-1696249987851 2023-10-02T12:33:20,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:33:21,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:33:21,141 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1696249987851,,1696249988357.73dd4d167cdf4d422164d1f17ee13a19. 2023-10-02T12:33:21,141 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=35 2023-10-02T12:33:21,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.HMaster(3995): Remote procedure done, pid=35 2023-10-02T12:33:21,142 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696250000457_default_test-1696249987851 on region 73dd4d167cdf4d422164d1f17ee13a19 2023-10-02T12:33:21,143 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 73dd4d167cdf4d422164d1f17ee13a19 2023-10-02T12:33:21,147 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=35, resume processing ppid=34 2023-10-02T12:33:21,147 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250000457_default_test-1696249987851 table=test-1696249987851 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:33:21,147 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=35, ppid=34, state=SUCCESS; SnapshotRegionProcedure 73dd4d167cdf4d422164d1f17ee13a19 in 637 msec 2023-10-02T12:33:21,148 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250000457_default_test-1696249987851 table=test-1696249987851 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:33:21,148 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250000457_default_test-1696249987851 table=test-1696249987851 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:33:21,148 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696250000457_default_test-1696249987851 2023-10-02T12:33:21,149 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/.hbase-snapshot/.tmp/snapshot_1696250000457_default_test-1696249987851 2023-10-02T12:33:21,436 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:33:21,573 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250000457_default_test-1696249987851 table=test-1696249987851 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:33:21,582 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250000457_default_test-1696249987851 table=test-1696249987851 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:33:21,583 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/.hbase-snapshot/.tmp/snapshot_1696250000457_default_test-1696249987851 to hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/.hbase-snapshot/snapshot_1696250000457_default_test-1696249987851 2023-10-02T12:33:21,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:33:21,585 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250000457_default_test-1696249987851 table=test-1696249987851 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:33:21,585 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696250000457_default_test-1696249987851 table=test-1696249987851 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-10-02T12:33:21,587 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=34, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250000457_default_test-1696249987851 table=test-1696249987851 type=FLUSH ttl=0 } in 1.1120 sec 2023-10-02T12:33:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:33:22,585 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: default:test-1696249987851, procId: 34 completed 2023-10-02T12:33:22,585 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1696249994339 2023-10-02T12:33:22,585 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-10-02T12:33:22,588 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-10-02T12:33:22,591 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1696249994339_test-1696249987851 2023-10-02T12:33:22,591 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1696250000457_default_test-1696249987851 to hdfs://localhost:46097/backupUT/backup_1696249994339/default/test-1696249987851/ 2023-10-02T12:33:22,606 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-10-02T12:33:22,647 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:45521, inputRoot=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560 2023-10-02T12:33:22,647 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,45341,1696249969200_-716783253_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:46097/backupUT/backup_1696249994339/default/test-1696249987851, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:46097/backupUT/backup_1696249994339/default/test-1696249987851/.hbase-snapshot/.tmp/snapshot_1696250000457_default_test-1696249987851 2023-10-02T12:33:22,647 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:45521, inputRoot=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560, snapshotDir=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/.hbase-snapshot/snapshot_1696250000457_default_test-1696249987851. 2023-10-02T12:33:22,660 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/.hbase-snapshot/snapshot_1696250000457_default_test-1696249987851 to hdfs://localhost:46097/backupUT/backup_1696249994339/default/test-1696249987851/.hbase-snapshot/.tmp/snapshot_1696250000457_default_test-1696249987851 2023-10-02T12:33:23,108 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:23,109 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:23,110 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:23,110 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:24,150 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/hadoop-7669046207768921785.jar 2023-10-02T12:33:24,150 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:24,151 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:24,152 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:24,152 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:24,153 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:24,153 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:24,153 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:24,154 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:24,154 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:24,155 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:33:24,156 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:33:24,157 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:33:24,157 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:33:24,158 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:33:24,158 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:33:24,159 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:33:24,159 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:33:24,160 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:33:24,161 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:33:24,161 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:33:24,162 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:33:24,166 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:33:24,166 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:33:24,167 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:33:24,167 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:33:24,168 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:33:24,169 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:33:24,169 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:33:24,233 WARN [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-10-02T12:33:24,266 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1696250000457_default_test-1696249987851' hfile list 2023-10-02T12:33:24,276 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-10-02T12:33:24,568 WARN [Listener at jenkins-hbase3.apache.org/46241 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/mapreduce.cluster.local.dir/1696250004486/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:33:24,568 WARN [Listener at jenkins-hbase3.apache.org/46241 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/mapreduce.cluster.local.dir/1696250004486/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:33:24,837 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-10-02T12:33:24,869 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/test-1696249987851/73dd4d167cdf4d422164d1f17ee13a19/f/8563d73b96cc4f0a84dda30b6f1539b2 output=hdfs://localhost:46097/backupUT/backup_1696249994339/default/test-1696249987851/archive/data/default/test-1696249987851/73dd4d167cdf4d422164d1f17ee13a19/f/8563d73b96cc4f0a84dda30b6f1539b2 2023-10-02T12:33:24,869 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 8.007M/sec 2023-10-02T12:33:25,722 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-10-02T12:33:25,723 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-10-02T12:33:25,736 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1696250000457_default_test-1696249987851 2023-10-02T12:33:25,736 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1696250000457_default_test-1696249987851 finished. 2023-10-02T12:33:25,737 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.BackupSystemTable(1034): test-1696249987851 2023-10-02T12:33:25,830 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696249994339 2023-10-02T12:33:25,830 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-10-02T12:33:26,257 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:46097/backupUT/backup_1696249994339/.backup.manifest 2023-10-02T12:33:26,258 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.TableBackupClient(358): Backup backup_1696249994339 finished: type=FULL,tablelist=test-1696249987851,targetRootDir=hdfs://localhost:46097/backupUT,startts=1696250000179,completets=1696250005826,bytescopied=0 2023-10-02T12:33:26,258 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-10-02T12:33:26,258 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1696250000457_default_test-1696249987851 2023-10-02T12:33:26,265 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696250000457_default_test-1696249987851" 2023-10-02T12:33:26,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696250000457_default_test-1696249987851 2023-10-02T12:33:26,271 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1696250000457_default_test-1696249987851 for backup backup_1696249994339 succeeded. 2023-10-02T12:33:26,272 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:33:26,285 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:33:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:33:26,289 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:33:26,296 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:33:26,321 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] impl.TableBackupClient(376): Backup backup_1696249994339 completed. 2023-10-02T12:33:26,321 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-02T12:33:26,321 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6c395121 to 127.0.0.1:49524 2023-10-02T12:33:26,322 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:33:26,322 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] backup.TestRemoteRestore(73): backup complete 2023-10-02T12:33:26,325 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] impl.RestoreTablesClient(101): HBase table table1 does not exist. It will be created during restore process 2023-10-02T12:33:26,326 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:46097/backupUT/backup_1696249994339/.backup.manifest 2023-10-02T12:33:26,329 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696249994339/.backup.manifest 2023-10-02T12:33:26,336 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:46097/backupUT/backup_1696249994339/.backup.manifest 2023-10-02T12:33:26,339 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696249994339/.backup.manifest 2023-10-02T12:33:26,339 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] impl.RestoreTablesClient(148): Restoring 'test-1696249987851' to 'table1' from full backup image hdfs://localhost:46097/backupUT/backup_1696249994339/default/test-1696249987851 2023-10-02T12:33:26,350 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] util.RestoreTool(487): Creating target table 'table1' 2023-10-02T12:33:26,351 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:46097/backupUT/backup_1696249994339/default/test-1696249987851/archive/data/default/test-1696249987851/73dd4d167cdf4d422164d1f17ee13a19 2023-10-02T12:33:26,352 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:46097/backupUT/backup_1696249994339/default/test-1696249987851/archive/data/default/test-1696249987851/73dd4d167cdf4d422164d1f17ee13a19/f in region [hdfs://localhost:46097/backupUT/backup_1696249994339/default/test-1696249987851/archive/data/default/test-1696249987851/73dd4d167cdf4d422164d1f17ee13a19] 2023-10-02T12:33:26,359 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] util.RestoreTool(442): Trying to figure out region boundaries hfile=hdfs://localhost:46097/backupUT/backup_1696249994339/default/test-1696249987851/archive/data/default/test-1696249987851/73dd4d167cdf4d422164d1f17ee13a19/f/8563d73b96cc4f0a84dda30b6f1539b2 first=row0 last=row98 2023-10-02T12:33:26,368 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:33:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] procedure2.ProcedureExecutor(1032): Stored pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=table1 2023-10-02T12:33:26,373 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:33:26,373 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 36 2023-10-02T12:33:26,374 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:33:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-10-02T12:33:26,385 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 0daec27bd9d108cbf260f6b50d5177c8, NAME => 'table1,,1696250006368.0daec27bd9d108cbf260f6b50d5177c8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560 2023-10-02T12:33:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-10-02T12:33:26,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-10-02T12:33:26,800 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(888): Instantiated table1,,1696250006368.0daec27bd9d108cbf260f6b50d5177c8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:26,800 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1668): Closing 0daec27bd9d108cbf260f6b50d5177c8, disabling compactions & flushes 2023-10-02T12:33:26,800 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1690): Closing region table1,,1696250006368.0daec27bd9d108cbf260f6b50d5177c8. 2023-10-02T12:33:26,800 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1696250006368.0daec27bd9d108cbf260f6b50d5177c8. 2023-10-02T12:33:26,801 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on table1,,1696250006368.0daec27bd9d108cbf260f6b50d5177c8. after waiting 0 ms 2023-10-02T12:33:26,801 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region table1,,1696250006368.0daec27bd9d108cbf260f6b50d5177c8. 2023-10-02T12:33:26,801 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1909): Closed table1,,1696250006368.0daec27bd9d108cbf260f6b50d5177c8. 2023-10-02T12:33:26,801 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1622): Region close journal for 0daec27bd9d108cbf260f6b50d5177c8: 2023-10-02T12:33:26,802 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:33:26,803 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"table1,,1696250006368.0daec27bd9d108cbf260f6b50d5177c8.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1696250006802"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696250006802"}]},"ts":"1696250006802"} 2023-10-02T12:33:26,806 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:33:26,807 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:33:26,807 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250006807"}]},"ts":"1696250006807"} 2023-10-02T12:33:26,809 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLING in hbase:meta 2023-10-02T12:33:26,817 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=0daec27bd9d108cbf260f6b50d5177c8, ASSIGN}] 2023-10-02T12:33:26,821 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=0daec27bd9d108cbf260f6b50d5177c8, ASSIGN 2023-10-02T12:33:26,822 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=table1, region=0daec27bd9d108cbf260f6b50d5177c8, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38375,1696249972520; forceNewPlan=false, retain=false 2023-10-02T12:33:26,975 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=0daec27bd9d108cbf260f6b50d5177c8, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:26,978 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE; OpenRegionProcedure 0daec27bd9d108cbf260f6b50d5177c8, server=jenkins-hbase3.apache.org,38375,1696249972520}] 2023-10-02T12:33:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-10-02T12:33:27,137 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(130): Open table1,,1696250006368.0daec27bd9d108cbf260f6b50d5177c8. 2023-10-02T12:33:27,137 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7254): Opening region: {ENCODED => 0daec27bd9d108cbf260f6b50d5177c8, NAME => 'table1,,1696250006368.0daec27bd9d108cbf260f6b50d5177c8.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:33:27,138 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:33:27,138 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 0daec27bd9d108cbf260f6b50d5177c8 2023-10-02T12:33:27,138 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(888): Instantiated table1,,1696250006368.0daec27bd9d108cbf260f6b50d5177c8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:27,138 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7296): checking encryption for 0daec27bd9d108cbf260f6b50d5177c8 2023-10-02T12:33:27,138 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7299): checking classloading for 0daec27bd9d108cbf260f6b50d5177c8 2023-10-02T12:33:27,146 INFO [StoreOpener-0daec27bd9d108cbf260f6b50d5177c8-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 0daec27bd9d108cbf260f6b50d5177c8 2023-10-02T12:33:27,149 INFO [StoreOpener-0daec27bd9d108cbf260f6b50d5177c8-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0daec27bd9d108cbf260f6b50d5177c8 columnFamilyName f 2023-10-02T12:33:27,150 DEBUG [StoreOpener-0daec27bd9d108cbf260f6b50d5177c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:27,150 INFO [StoreOpener-0daec27bd9d108cbf260f6b50d5177c8-1 {}] regionserver.HStore(324): Store=0daec27bd9d108cbf260f6b50d5177c8/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:33:27,152 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/table1/0daec27bd9d108cbf260f6b50d5177c8 2023-10-02T12:33:27,153 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/table1/0daec27bd9d108cbf260f6b50d5177c8 2023-10-02T12:33:27,157 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1076): writing seq id for 0daec27bd9d108cbf260f6b50d5177c8 2023-10-02T12:33:27,160 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/table1/0daec27bd9d108cbf260f6b50d5177c8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:33:27,161 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): Opened 0daec27bd9d108cbf260f6b50d5177c8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11950488800, jitterRate=0.11297599971294403}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:33:27,162 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(992): Region open journal for 0daec27bd9d108cbf260f6b50d5177c8: 2023-10-02T12:33:27,163 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2556): Post open deploy tasks for table1,,1696250006368.0daec27bd9d108cbf260f6b50d5177c8., pid=38, masterSystemTime=1696250007131 2023-10-02T12:33:27,165 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2583): Finished post open deploy task for table1,,1696250006368.0daec27bd9d108cbf260f6b50d5177c8. 2023-10-02T12:33:27,165 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(158): Opened table1,,1696250006368.0daec27bd9d108cbf260f6b50d5177c8. 2023-10-02T12:33:27,166 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=0daec27bd9d108cbf260f6b50d5177c8, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:27,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=38, resume processing ppid=37 2023-10-02T12:33:27,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=38, ppid=37, state=SUCCESS; OpenRegionProcedure 0daec27bd9d108cbf260f6b50d5177c8, server=jenkins-hbase3.apache.org,38375,1696249972520 in 190 msec 2023-10-02T12:33:27,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=37, resume processing ppid=36 2023-10-02T12:33:27,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=37, ppid=36, state=SUCCESS; TransitRegionStateProcedure table=table1, region=0daec27bd9d108cbf260f6b50d5177c8, ASSIGN in 354 msec 2023-10-02T12:33:27,174 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:33:27,175 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250007174"}]},"ts":"1696250007174"} 2023-10-02T12:33:27,176 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLED in hbase:meta 2023-10-02T12:33:27,179 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:33:27,182 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=36, state=SUCCESS; CreateTableProcedure table=table1 in 811 msec 2023-10-02T12:33:27,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-10-02T12:33:27,481 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:table1, procId: 36 completed 2023-10-02T12:33:27,488 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:46097/backupUT/backup_1696249994339/default/test-1696249987851/archive/data/default/test-1696249987851/73dd4d167cdf4d422164d1f17ee13a19 from hbase tables test-1696249987851 to tables table1 2023-10-02T12:33:27,488 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.MapReduceRestoreJob(72): Restore test-1696249987851 into table1 2023-10-02T12:33:27,492 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/hbase-staging/bulk_output-default-table1-1696250007488 from hdfs://localhost:46097/backupUT/backup_1696249994339/default/test-1696249987851/archive/data/default/test-1696249987851/73dd4d167cdf4d422164d1f17ee13a19 2023-10-02T12:33:27,497 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5849e66a to 127.0.0.1:49524 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:33:27,503 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54878213, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:33:27,505 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:33:27,507 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48370, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:33:27,522 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-10-02T12:33:27,523 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table table1 2023-10-02T12:33:27,528 DEBUG [hconnection-0x515afb36-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:33:27,530 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46922, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:33:27,533 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for table1: 2023-10-02T12:33:27,534 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-10-02T12:33:27,534 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/hbase-staging/partitions_e2c7ef2e-8f24-4d01-8e8e-a9838c2d48aa 2023-10-02T12:33:27,580 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:27,580 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:27,581 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:27,582 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:27,662 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-10-02T12:33:27,664 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48380, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-10-02T12:33:28,788 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/hadoop-2161517595436151732.jar 2023-10-02T12:33:28,789 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:28,789 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:28,790 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:28,790 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:28,791 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:28,791 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:28,792 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:28,792 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:28,793 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:28,794 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:33:28,794 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:33:28,795 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:33:28,795 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:33:28,796 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:33:28,797 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:33:28,797 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:33:28,798 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:33:28,798 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:33:28,799 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:33:28,799 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:33:28,800 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:33:28,804 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:28,805 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:28,805 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:28,806 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:28,806 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:28,806 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:28,807 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:33:28,807 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: table1 2023-10-02T12:33:28,808 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-02T12:33:28,808 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5849e66a to 127.0.0.1:49524 2023-10-02T12:33:28,808 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:33:28,808 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-10-02T12:33:28,808 WARN [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-10-02T12:33:28,809 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:33:28,816 WARN [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-10-02T12:33:28,824 WARN [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-10-02T12:33:28,943 WARN [Listener at jenkins-hbase3.apache.org/46241 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/mapreduce.cluster.local.dir/1696250008889/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:33:28,943 WARN [Listener at jenkins-hbase3.apache.org/46241 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/mapreduce.cluster.local.dir/1696250008889/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:33:29,037 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:46097/backupUT/backup_1696249994339/default/test-1696249987851/archive/data/default/test-1696249987851/73dd4d167cdf4d422164d1f17ee13a19/f/8563d73b96cc4f0a84dda30b6f1539b2 2023-10-02T12:33:29,043 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-10-02T12:33:29,206 DEBUG [pool-322-thread-1 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x498e3261 to 127.0.0.1:49524 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:33:29,219 DEBUG [pool-322-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5df70887, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:33:29,230 DEBUG [hconnection-0x6caf995e-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:33:29,232 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46930, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:33:29,237 DEBUG [pool-322-thread-1 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x498e3261 to 127.0.0.1:49524 2023-10-02T12:33:29,237 DEBUG [pool-322-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:33:29,240 DEBUG [pool-322-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-10-02T12:33:29,240 DEBUG [pool-322-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-10-02T12:33:29,279 DEBUG [pool-322-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/hbase-staging/bulk_output-default-table1-1696250007488/_temporary/0/_temporary/attempt_local1862991897_0002_r_000000_0/f/6ceedea74bd64965b5473b7a88b41b75 is 37, key is row10/f:q1/1696249990530/Put/seqid=0 2023-10-02T12:33:29,282 WARN [Thread-968 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:38375] Specified favored nodes: [jenkins-hbase3.apache.org:38375] 2023-10-02T12:33:30,034 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/hbase-staging/bulk_output-default-table1-1696250007488 2023-10-02T12:33:30,035 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x05772783 to 127.0.0.1:49524 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:33:30,052 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66ffd1c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:33:30,054 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:33:30,058 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46944, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:33:30,071 WARN [Listener at jenkins-hbase3.apache.org/46241 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/hbase-staging/bulk_output-default-table1-1696250007488/_SUCCESS 2023-10-02T12:33:30,077 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:33:30,080 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48390, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:33:30,123 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/hbase-staging/bulk_output-default-table1-1696250007488/f/6ceedea74bd64965b5473b7a88b41b75 first=Optional[row0] last=Optional[row98] 2023-10-02T12:33:30,153 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=table1,,1696250006368.0daec27bd9d108cbf260f6b50d5177c8., hostname=jenkins-hbase3.apache.org,38375,1696249972520, seqNum=2 for row with hfile group [{f,hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/hbase-staging/bulk_output-default-table1-1696250007488/f/6ceedea74bd64965b5473b7a88b41b75}] 2023-10-02T12:33:30,183 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/hbase-staging/bulk_output-default-table1-1696250007488/f/6ceedea74bd64965b5473b7a88b41b75 for inclusion in 0daec27bd9d108cbf260f6b50d5177c8/f 2023-10-02T12:33:30,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-10-02T12:33:30,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] regionserver.HStore(625): Region bounds: first= last= 2023-10-02T12:33:30,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] regionserver.HRegion(2520): Flush status journal for 0daec27bd9d108cbf260f6b50d5177c8: 2023-10-02T12:33:30,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/hbase-staging/bulk_output-default-table1-1696250007488/f/6ceedea74bd64965b5473b7a88b41b75 to hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/staging/jenkins__table1__st0huq2e1ulcft2gorg2tt0q52rk9utogcsut9jr0j0pl1k3big1c0a4cgi4qg4p/f/6ceedea74bd64965b5473b7a88b41b75 2023-10-02T12:33:30,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/staging/jenkins__table1__st0huq2e1ulcft2gorg2tt0q52rk9utogcsut9jr0j0pl1k3big1c0a4cgi4qg4p/f/6ceedea74bd64965b5473b7a88b41b75 as hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/table1/0daec27bd9d108cbf260f6b50d5177c8/f/71769f45d4fb468bad3b8c1ae8074630_SeqId_4_ 2023-10-02T12:33:30,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x06a2e6ec to 127.0.0.1:49524 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:33:30,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56be8268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:33:30,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:33:30,247 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:58282, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:33:30,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:33:30,271 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:39746, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:33:30,304 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-02T12:33:30,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x06a2e6ec to 127.0.0.1:49524 2023-10-02T12:33:30,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:33:30,306 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/staging/jenkins__table1__st0huq2e1ulcft2gorg2tt0q52rk9utogcsut9jr0j0pl1k3big1c0a4cgi4qg4p/f/6ceedea74bd64965b5473b7a88b41b75 into 0daec27bd9d108cbf260f6b50d5177c8/f as hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/table1/0daec27bd9d108cbf260f6b50d5177c8/f/71769f45d4fb468bad3b8c1ae8074630_SeqId_4_ - updating store file list. 2023-10-02T12:33:30,315 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/table1/0daec27bd9d108cbf260f6b50d5177c8/f/71769f45d4fb468bad3b8c1ae8074630_SeqId_4_ into 0daec27bd9d108cbf260f6b50d5177c8/f 2023-10-02T12:33:30,315 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/staging/jenkins__table1__st0huq2e1ulcft2gorg2tt0q52rk9utogcsut9jr0j0pl1k3big1c0a4cgi4qg4p/f/6ceedea74bd64965b5473b7a88b41b75 into 0daec27bd9d108cbf260f6b50d5177c8/f (new location: hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/table1/0daec27bd9d108cbf260f6b50d5177c8/f/71769f45d4fb468bad3b8c1ae8074630_SeqId_4_) 2023-10-02T12:33:30,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/staging/jenkins__table1__st0huq2e1ulcft2gorg2tt0q52rk9utogcsut9jr0j0pl1k3big1c0a4cgi4qg4p/f/6ceedea74bd64965b5473b7a88b41b75 2023-10-02T12:33:30,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x02781be7 to 127.0.0.1:49524 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:33:30,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c0f6a45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:33:30,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:33:30,355 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:58292, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-10-02T12:33:30,355 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2023-10-02T12:33:30,355 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_table1 Metrics about Tables on a single HBase RegionServer 2023-10-02T12:33:30,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:33:30,368 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:39752, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-10-02T12:33:30,404 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-02T12:33:30,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x02781be7 to 127.0.0.1:49524 2023-10-02T12:33:30,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:33:30,426 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-02T12:33:30,426 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x05772783 to 127.0.0.1:49524 2023-10-02T12:33:30,426 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:33:30,428 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-10-02T12:33:30,428 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-10-02T12:33:30,428 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] impl.RestoreTablesClient(233): Backup: backup_1696249994339 hdfs://localhost:46097/backupUT/backup_1696249994339/default/test-1696249987851/ 2023-10-02T12:33:30,428 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-10-02T12:33:30,431 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] client.HBaseAdmin$18(967): Started disable of table1 2023-10-02T12:33:30,437 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable table1 2023-10-02T12:33:30,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] procedure2.ProcedureExecutor(1032): Stored pid=39, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=table1 2023-10-02T12:33:30,454 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250010454"}]},"ts":"1696250010454"} 2023-10-02T12:33:30,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=39 2023-10-02T12:33:30,457 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLING in hbase:meta 2023-10-02T12:33:30,459 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(293): Set table1 to state=DISABLING 2023-10-02T12:33:30,461 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=0daec27bd9d108cbf260f6b50d5177c8, UNASSIGN}] 2023-10-02T12:33:30,464 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=0daec27bd9d108cbf260f6b50d5177c8, UNASSIGN 2023-10-02T12:33:30,465 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=0daec27bd9d108cbf260f6b50d5177c8, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:30,467 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE; CloseRegionProcedure 0daec27bd9d108cbf260f6b50d5177c8, server=jenkins-hbase3.apache.org,38375,1696249972520}] 2023-10-02T12:33:30,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=39 2023-10-02T12:33:30,646 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] handler.UnassignRegionHandler(119): Close 0daec27bd9d108cbf260f6b50d5177c8 2023-10-02T12:33:30,646 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1668): Closing 0daec27bd9d108cbf260f6b50d5177c8, disabling compactions & flushes 2023-10-02T12:33:30,646 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1690): Closing region table1,,1696250006368.0daec27bd9d108cbf260f6b50d5177c8. 2023-10-02T12:33:30,646 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1696250006368.0daec27bd9d108cbf260f6b50d5177c8. 2023-10-02T12:33:30,646 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1778): Acquired close lock on table1,,1696250006368.0daec27bd9d108cbf260f6b50d5177c8. after waiting 0 ms 2023-10-02T12:33:30,646 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1788): Updates disabled for region table1,,1696250006368.0daec27bd9d108cbf260f6b50d5177c8. 2023-10-02T12:33:30,657 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/table1/0daec27bd9d108cbf260f6b50d5177c8/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-10-02T12:33:30,658 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:33:30,658 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1909): Closed table1,,1696250006368.0daec27bd9d108cbf260f6b50d5177c8. 2023-10-02T12:33:30,658 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1622): Region close journal for 0daec27bd9d108cbf260f6b50d5177c8: 2023-10-02T12:33:30,662 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] handler.UnassignRegionHandler(163): Closed 0daec27bd9d108cbf260f6b50d5177c8 2023-10-02T12:33:30,663 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=0daec27bd9d108cbf260f6b50d5177c8, regionState=CLOSED 2023-10-02T12:33:30,667 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=41, resume processing ppid=40 2023-10-02T12:33:30,667 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=41, ppid=40, state=SUCCESS; CloseRegionProcedure 0daec27bd9d108cbf260f6b50d5177c8, server=jenkins-hbase3.apache.org,38375,1696249972520 in 197 msec 2023-10-02T12:33:30,669 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=40, resume processing ppid=39 2023-10-02T12:33:30,669 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=40, ppid=39, state=SUCCESS; TransitRegionStateProcedure table=table1, region=0daec27bd9d108cbf260f6b50d5177c8, UNASSIGN in 206 msec 2023-10-02T12:33:30,670 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250010669"}]},"ts":"1696250010669"} 2023-10-02T12:33:30,671 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLED in hbase:meta 2023-10-02T12:33:30,673 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set table1 to state=DISABLED 2023-10-02T12:33:30,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=39, state=SUCCESS; DisableTableProcedure table=table1 in 234 msec 2023-10-02T12:33:30,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=39 2023-10-02T12:33:30,760 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] client.HBaseAdmin$TableFuture(3677): Operation: DISABLE, Table Name: default:table1, procId: 39 completed 2023-10-02T12:33:30,767 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.HMaster$5(2448): Client=jenkins//172.31.12.81 delete table1 2023-10-02T12:33:30,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] procedure2.ProcedureExecutor(1032): Stored pid=42, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=table1 2023-10-02T12:33:30,782 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=42, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-10-02T12:33:30,784 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=42, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=table1 2023-10-02T12:33:30,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-10-02T12:33:30,792 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/table1/0daec27bd9d108cbf260f6b50d5177c8 2023-10-02T12:33:30,797 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/table1/0daec27bd9d108cbf260f6b50d5177c8/f, FileablePath, hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/table1/0daec27bd9d108cbf260f6b50d5177c8/recovered.edits] 2023-10-02T12:33:30,807 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/table1/0daec27bd9d108cbf260f6b50d5177c8/f/71769f45d4fb468bad3b8c1ae8074630_SeqId_4_ to hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/archive/data/default/table1/0daec27bd9d108cbf260f6b50d5177c8/f/71769f45d4fb468bad3b8c1ae8074630_SeqId_4_ 2023-10-02T12:33:30,812 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/table1/0daec27bd9d108cbf260f6b50d5177c8/recovered.edits/6.seqid to hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/archive/data/default/table1/0daec27bd9d108cbf260f6b50d5177c8/recovered.edits/6.seqid 2023-10-02T12:33:30,813 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/table1/0daec27bd9d108cbf260f6b50d5177c8 2023-10-02T12:33:30,813 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(293): Archived table1 regions 2023-10-02T12:33:30,817 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=42, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=table1 2023-10-02T12:33:30,828 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of table1 from hbase:meta 2023-10-02T12:33:30,832 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(387): Removing 'table1' descriptor. 2023-10-02T12:33:30,833 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=42, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=table1 2023-10-02T12:33:30,833 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(377): Removing 'table1' from region states. 2023-10-02T12:33:30,834 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1,,1696250006368.0daec27bd9d108cbf260f6b50d5177c8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1696250010833"}]},"ts":"9223372036854775807"} 2023-10-02T12:33:30,837 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-10-02T12:33:30,837 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 0daec27bd9d108cbf260f6b50d5177c8, NAME => 'table1,,1696250006368.0daec27bd9d108cbf260f6b50d5177c8.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:33:30,837 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(381): Marking 'table1' as deleted. 2023-10-02T12:33:30,837 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1696250010837"}]},"ts":"9223372036854775807"} 2023-10-02T12:33:30,840 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1678): Deleted table table1 state from META 2023-10-02T12:33:30,844 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(127): Finished pid=42, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-10-02T12:33:30,846 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=42, state=SUCCESS; DeleteTableProcedure table=table1 in 75 msec 2023-10-02T12:33:30,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-10-02T12:33:30,889 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] client.HBaseAdmin$TableFuture(3677): Operation: DELETE, Table Name: default:table1, procId: 42 completed 2023-10-02T12:33:30,919 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.ResourceChecker(175): after: backup.TestRemoteRestore#testFullRestoreRemote Thread=830 (was 815) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-356495742_16 at /127.0.0.1:54270 [Waiting for operation #8] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: HFileArchiver-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1259401672_16 at /127.0.0.1:50394 [Receiving block BP-2107663488-172.31.12.81-1696249963873:blk_1073741856_1032] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: member: 'jenkins-hbase3.apache.org,38375,1696249972520' subprocedure-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,38375,1696249972520)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1259401672_16 at /127.0.0.1:50382 [Receiving block BP-2107663488-172.31.12.81-1696249963873:blk_1073741855_1031] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_109360846_16 at /127.0.0.1:50426 [Waiting for operation #14] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/cluster_f3ba57df-0a4a-7a4e-2411-dac870a6ebff/dfs/data/data1/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x46f0727e-shared-pool-10 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x46f0727e-shared-pool-8 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/cluster_f3ba57df-0a4a-7a4e-2411-dac870a6ebff/dfs/data/data2/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x46f0727e-shared-pool-6 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-2107663488-172.31.12.81-1696249963873:blk_1073741856_1032, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x46f0727e-shared-pool-7 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x46f0727e-shared-pool-11 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: (jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-2107663488-172.31.12.81-1696249963873:blk_1073741855_1031, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Client (556141606) connection to localhost/127.0.0.1:46097 from jenkins.hfs.1 java.lang.Object.wait(Native Method) org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1053) org.apache.hadoop.ipc.Client$Connection.run(Client.java:1097) Potentially hanging thread: hconnection-0x46f0727e-shared-pool-9 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=903 (was 878) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=447 (was 471), ProcessCount=181 (was 181), AvailableMemoryMB=2243 (was 1694) - AvailableMemoryMB LEAK? - 2023-10-02T12:33:30,919 WARN [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.ResourceChecker(130): Thread=830 is superior to 500 2023-10-02T12:33:30,945 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.ResourceChecker(147): before: backup.TestRemoteRestore#testFullRestoreRemoteWithAlternateRestoreOutputDir Thread=830, OpenFileDescriptor=903, MaxFileDescriptor=60000, SystemLoadAverage=447, ProcessCount=181, AvailableMemoryMB=2242 2023-10-02T12:33:30,945 WARN [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.ResourceChecker(130): Thread=830 is superior to 500 2023-10-02T12:33:30,945 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] backup.TestRemoteRestore(91): test remote full backup on a single table with alternate restore output dir 2023-10-02T12:33:30,947 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5a40feff to 127.0.0.1:49524 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:33:30,959 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@81d0e4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:33:30,963 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:33:30,966 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:39754, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:33:30,968 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:33:30,971 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:58300, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:33:30,992 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:33:31,000 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:33:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250011000 (current time:1696250011000). 2023-10-02T12:33:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:33:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:33:31,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x753d8779 to 127.0.0.1:49524 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:33:31,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29b0f38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:33:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:33:31,020 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:39758, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:33:31,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x753d8779 to 127.0.0.1:49524 2023-10-02T12:33:31,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:33:31,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:33:31,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] procedure2.ProcedureExecutor(1032): Stored pid=43, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:33:31,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 43 2023-10-02T12:33:31,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=43 2023-10-02T12:33:31,030 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:33:31,031 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:33:31,035 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/.hbase-snapshot, current working dir path: hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor136.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor136.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:33:31,038 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:33:31,049 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:33:31,050 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE; SnapshotRegionProcedure 7fa9a98238e4b256d46f306c833fe67e}] 2023-10-02T12:33:31,053 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=44, ppid=43, state=RUNNABLE; SnapshotRegionProcedure 7fa9a98238e4b256d46f306c833fe67e 2023-10-02T12:33:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=43 2023-10-02T12:33:31,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38375 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=44 2023-10-02T12:33:31,206 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696249994709.7fa9a98238e4b256d46f306c833fe67e. 2023-10-02T12:33:31,206 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegion(2819): Flushing 7fa9a98238e4b256d46f306c833fe67e 2/2 column families, dataSize=1.08 KB heapSize=2.23 KB 2023-10-02T12:33:31,228 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/.tmp/meta/def7cd9f351940989a275163003fe440 is 169, key is trslm:hdfs://localhost:46097/backupUT\x00test-1696249987851/meta:log-roll-map/1696250005810/Put/seqid=0 2023-10-02T12:33:31,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=43 2023-10-02T12:33:31,355 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-10-02T12:33:31,359 INFO [RS-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:47006, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2023-10-02T12:33:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=43 2023-10-02T12:33:31,636 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=526 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/.tmp/meta/def7cd9f351940989a275163003fe440 2023-10-02T12:33:31,659 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/.tmp/session/a8d57579ebc740a2ac456ae0080d69ce is 310, key is session:backup_1696249994339/session:context/1696250006295/Put/seqid=0 2023-10-02T12:33:32,094 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=579 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/.tmp/session/a8d57579ebc740a2ac456ae0080d69ce 2023-10-02T12:33:32,103 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/.tmp/meta/def7cd9f351940989a275163003fe440 as hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/meta/def7cd9f351940989a275163003fe440 2023-10-02T12:33:32,112 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/meta/def7cd9f351940989a275163003fe440, entries=4, sequenceid=17, filesize=5.5 K 2023-10-02T12:33:32,113 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/.tmp/session/a8d57579ebc740a2ac456ae0080d69ce as hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/session/a8d57579ebc740a2ac456ae0080d69ce 2023-10-02T12:33:32,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=43 2023-10-02T12:33:32,138 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/session/a8d57579ebc740a2ac456ae0080d69ce, entries=2, sequenceid=17, filesize=5.3 K 2023-10-02T12:33:32,140 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegion(3022): Finished flush of dataSize ~1.08 KB/1105, heapSize ~2.20 KB/2256, currentSize=0 B/0 for 7fa9a98238e4b256d46f306c833fe67e in 933ms, sequenceid=17, compaction requested=false 2023-10-02T12:33:32,140 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegion(2520): Flush status journal for 7fa9a98238e4b256d46f306c833fe67e: 2023-10-02T12:33:32,140 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696249994709.7fa9a98238e4b256d46f306c833fe67e. for snapshot_backup_system completed. 2023-10-02T12:33:32,140 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696249994709.7fa9a98238e4b256d46f306c833fe67e.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:33:32,140 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:33:32,140 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/meta/def7cd9f351940989a275163003fe440] hfiles 2023-10-02T12:33:32,140 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/meta/def7cd9f351940989a275163003fe440 for snapshot=snapshot_backup_system 2023-10-02T12:33:32,140 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/session/cdf06e31c7274ccc94f7501d8ddd6b22, hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/session/a8d57579ebc740a2ac456ae0080d69ce] hfiles 2023-10-02T12:33:32,140 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/session/cdf06e31c7274ccc94f7501d8ddd6b22 for snapshot=snapshot_backup_system 2023-10-02T12:33:32,140 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/session/a8d57579ebc740a2ac456ae0080d69ce for snapshot=snapshot_backup_system 2023-10-02T12:33:32,568 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696249994709.7fa9a98238e4b256d46f306c833fe67e. 2023-10-02T12:33:32,568 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=44 2023-10-02T12:33:32,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.HMaster(3995): Remote procedure done, pid=44 2023-10-02T12:33:32,569 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 7fa9a98238e4b256d46f306c833fe67e 2023-10-02T12:33:32,571 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=44, ppid=43, state=RUNNABLE; SnapshotRegionProcedure 7fa9a98238e4b256d46f306c833fe67e 2023-10-02T12:33:32,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=44, resume processing ppid=43 2023-10-02T12:33:32,577 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:33:32,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=44, ppid=43, state=SUCCESS; SnapshotRegionProcedure 7fa9a98238e4b256d46f306c833fe67e in 1.5220 sec 2023-10-02T12:33:32,578 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:33:32,580 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:33:32,580 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:33:32,581 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:33:32,786 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:33:33,000 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:33:33,009 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:33:33,009 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:33:33,011 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:33:33,011 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 43 2023-10-02T12:33:33,012 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=43, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.9880 sec 2023-10-02T12:33:33,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=43 2023-10-02T12:33:33,138 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: backup:system, procId: 43 completed 2023-10-02T12:33:33,139 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] impl.TableBackupClient(115): Backup backup_1696250010959 started at 1696250013139. 2023-10-02T12:33:33,141 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.TableBackupClient(119): Backup session backup_1696250010959 has been started. 2023-10-02T12:33:33,152 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-10-02T12:33:33,158 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-10-02T12:33:33,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-10-02T12:33:33,159 INFO [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-10-02T12:33:33,159 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-10-02T12:33:33,161 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-10-02T12:33:33,161 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-10-02T12:33:33,161 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-10-02T12:33:33,161 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:33:33,164 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:33,164 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-10-02T12:33:33,164 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:33:33,164 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:33:33,164 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:33,165 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-10-02T12:33:33,165 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-10-02T12:33:33,165 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-10-02T12:33:33,165 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-10-02T12:33:33,165 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:33:33,165 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-10-02T12:33:33,166 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-10-02T12:33:33,166 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-10-02T12:33:33,166 DEBUG [member: 'jenkins-hbase3.apache.org,38375,1696249972520' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-10-02T12:33:33,166 DEBUG [member: 'jenkins-hbase3.apache.org,38375,1696249972520' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-10-02T12:33:33,167 DEBUG [member: 'jenkins-hbase3.apache.org,38375,1696249972520' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-10-02T12:33:33,167 DEBUG [member: 'jenkins-hbase3.apache.org,38375,1696249972520' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-10-02T12:33:33,167 DEBUG [member: 'jenkins-hbase3.apache.org,38375,1696249972520' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,38375,1696249972520' joining acquired barrier for procedure (rolllog) in zk 2023-10-02T12:33:33,169 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:33,169 DEBUG [member: 'jenkins-hbase3.apache.org,38375,1696249972520' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-10-02T12:33:33,169 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:33,169 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:33:33,169 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:33:33,170 DEBUG [member: 'jenkins-hbase3.apache.org,38375,1696249972520' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-10-02T12:33:33,170 DEBUG [member: 'jenkins-hbase3.apache.org,38375,1696249972520' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-10-02T12:33:33,170 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:33:33,170 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:33:33,171 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:33:33,171 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:33,172 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:33:33,173 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,38375,1696249972520' joining acquired barrier for procedure 'rolllog' on coordinator 2023-10-02T12:33:33,173 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@6631cbc6[Count = 0] remaining members to acquire global barrier 2023-10-02T12:33:33,173 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-10-02T12:33:33,173 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-10-02T12:33:33,175 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:33:33,175 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-10-02T12:33:33,175 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-10-02T12:33:33,176 DEBUG [member: 'jenkins-hbase3.apache.org,38375,1696249972520' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-10-02T12:33:33,176 DEBUG [member: 'jenkins-hbase3.apache.org,38375,1696249972520' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-10-02T12:33:33,176 DEBUG [rs(jenkins-hbase3.apache.org,38375,1696249972520)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:33,176 INFO [rs(jenkins-hbase3.apache.org,38375,1696249972520)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1696250000242 highest: 1696250000242 on jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:33,176 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C38375%2C1696249972520:(num 1696250000242) roll requested 2023-10-02T12:33:33,182 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:33,182 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-10-02T12:33:33,194 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/WALs/jenkins-hbase3.apache.org,38375,1696249972520/jenkins-hbase3.apache.org%2C38375%2C1696249972520.1696250013176, exclude list is [], retry=0 2023-10-02T12:33:33,201 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43099,DS-c62d647f-3988-4c56-a813-dcabca820155,DISK] 2023-10-02T12:33:33,204 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/WALs/jenkins-hbase3.apache.org,38375,1696249972520/jenkins-hbase3.apache.org%2C38375%2C1696249972520.1696250000242 with entries=15, filesize=3.47 KB; new WAL /user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/WALs/jenkins-hbase3.apache.org,38375,1696249972520/jenkins-hbase3.apache.org%2C38375%2C1696249972520.1696250013176 2023-10-02T12:33:33,204 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43099,DS-c62d647f-3988-4c56-a813-dcabca820155,DISK]] 2023-10-02T12:33:33,204 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/WALs/jenkins-hbase3.apache.org,38375,1696249972520/jenkins-hbase3.apache.org%2C38375%2C1696249972520.1696250000242 is not closed yet, will try archiving it next time 2023-10-02T12:33:33,204 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C38375%2C1696249972520.meta:.meta(num 1696250000274) roll requested 2023-10-02T12:33:33,224 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/WALs/jenkins-hbase3.apache.org,38375,1696249972520/jenkins-hbase3.apache.org%2C38375%2C1696249972520.meta.1696250013204.meta, exclude list is [], retry=0 2023-10-02T12:33:33,238 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43099,DS-c62d647f-3988-4c56-a813-dcabca820155,DISK] 2023-10-02T12:33:33,240 DEBUG [rs(jenkins-hbase3.apache.org,38375,1696249972520)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 64 2023-10-02T12:33:33,240 INFO [rs(jenkins-hbase3.apache.org,38375,1696249972520)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1696250013176 on jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:33,241 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/WALs/jenkins-hbase3.apache.org,38375,1696249972520/jenkins-hbase3.apache.org%2C38375%2C1696249972520.meta.1696250000274.meta with entries=12, filesize=2.73 KB; new WAL /user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/WALs/jenkins-hbase3.apache.org,38375,1696249972520/jenkins-hbase3.apache.org%2C38375%2C1696249972520.meta.1696250013204.meta 2023-10-02T12:33:33,241 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43099,DS-c62d647f-3988-4c56-a813-dcabca820155,DISK]] 2023-10-02T12:33:33,241 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/WALs/jenkins-hbase3.apache.org,38375,1696249972520/jenkins-hbase3.apache.org%2C38375%2C1696249972520.meta.1696250000274.meta is not closed yet, will try archiving it next time 2023-10-02T12:33:33,255 DEBUG [member: 'jenkins-hbase3.apache.org,38375,1696249972520' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-10-02T12:33:33,255 DEBUG [member: 'jenkins-hbase3.apache.org,38375,1696249972520' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,38375,1696249972520' in zk 2023-10-02T12:33:33,257 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:33,257 DEBUG [member: 'jenkins-hbase3.apache.org,38375,1696249972520' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-10-02T12:33:33,257 DEBUG [member: 'jenkins-hbase3.apache.org,38375,1696249972520' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:33:33,257 DEBUG [member: 'jenkins-hbase3.apache.org,38375,1696249972520' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-10-02T12:33:33,258 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:33,260 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:33:33,260 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:33:33,260 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:33:33,261 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:33:33,261 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:33:33,266 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:33,266 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:33:33,267 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:33:33,267 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:33,268 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,38375,1696249972520': 2023-10-02T12:33:33,268 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,38375,1696249972520' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-10-02T12:33:33,268 INFO [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-10-02T12:33:33,268 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-10-02T12:33:33,268 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-10-02T12:33:33,268 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-10-02T12:33:33,268 INFO [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-10-02T12:33:33,273 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:33:33,273 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:33:33,273 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-10-02T12:33:33,273 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-10-02T12:33:33,273 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:33:33,273 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-10-02T12:33:33,273 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:33:33,273 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:33:33,274 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:33:33,274 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:33:33,274 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:33:33,274 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:33:33,274 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:33,275 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:33:33,275 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:33:33,275 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:33:33,276 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:33,276 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:33,276 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:33:33,277 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:33:33,277 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:33,280 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-10-02T12:33:33,280 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-10-02T12:33:33,280 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:33:33,280 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:33:33,281 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:33,281 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-10-02T12:33:33,281 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-10-02T12:33:33,281 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:33,281 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:33:33,281 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:33:33,281 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:33:33,281 DEBUG [(jenkins-hbase3.apache.org,45341,1696249969200)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:33:33,281 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-10-02T12:33:33,283 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-10-02T12:33:33,284 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] client.HBaseAdmin(2838): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-10-02T12:33:33,284 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] client.HBaseAdmin(2846): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-10-02T12:33:33,284 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:33:33,284 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:33:33,384 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] client.HBaseAdmin(2852): Getting current status of procedure from master... 2023-10-02T12:33:33,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-10-02T12:33:33,391 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696250013389_default_test-1696249987851 table=test-1696249987851 type=FLUSH ttl=0 } 2023-10-02T12:33:33,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250013391 (current time:1696250013391). 2023-10-02T12:33:33,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:33:33,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696250013389_default_test-1696249987851 VERSION not specified, setting to 2 2023-10-02T12:33:33,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x23e6e2c7 to 127.0.0.1:49524 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:33:33,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@156346cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:33:33,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:33:33,419 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:39762, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:33:33,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x23e6e2c7 to 127.0.0.1:49524 2023-10-02T12:33:33,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:33:33,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:33:33,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] procedure2.ProcedureExecutor(1032): Stored pid=45, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1696250013389_default_test-1696249987851 table=test-1696249987851 type=FLUSH ttl=0 } 2023-10-02T12:33:33,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696250013389_default_test-1696249987851 table=test-1696249987851 type=FLUSH ttl=0 }, snapshot procedure id = 45 2023-10-02T12:33:33,428 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1696250013389_default_test-1696249987851 table=test-1696249987851 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:33:33,429 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1696250013389_default_test-1696249987851 table=test-1696249987851 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:33:33,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=45 2023-10-02T12:33:33,448 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/.hbase-snapshot, current working dir path: hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/.hbase-snapshot/.tmp/snapshot_1696250013389_default_test-1696249987851 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor136.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor136.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor136.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor136.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor136.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor136.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:33:33,451 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1696250013389_default_test-1696249987851 table=test-1696249987851 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:33:33,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=45 2023-10-02T12:33:33,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=45 2023-10-02T12:33:33,862 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1696250013389_default_test-1696249987851 table=test-1696249987851 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:33:33,862 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE; SnapshotRegionProcedure 73dd4d167cdf4d422164d1f17ee13a19}] 2023-10-02T12:33:33,865 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=46, ppid=45, state=RUNNABLE; SnapshotRegionProcedure 73dd4d167cdf4d422164d1f17ee13a19 2023-10-02T12:33:34,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38375 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=46 2023-10-02T12:33:34,016 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1696249987851,,1696249988357.73dd4d167cdf4d422164d1f17ee13a19. 2023-10-02T12:33:34,016 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] regionserver.HRegion(2520): Flush status journal for 73dd4d167cdf4d422164d1f17ee13a19: 2023-10-02T12:33:34,016 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1696249987851,,1696249988357.73dd4d167cdf4d422164d1f17ee13a19. for snapshot_1696250013389_default_test-1696249987851 completed. 2023-10-02T12:33:34,017 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] snapshot.SnapshotManifest(240): Storing 'test-1696249987851,,1696249988357.73dd4d167cdf4d422164d1f17ee13a19.' region-info for snapshot=snapshot_1696250013389_default_test-1696249987851 2023-10-02T12:33:34,017 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:33:34,017 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/test-1696249987851/73dd4d167cdf4d422164d1f17ee13a19/f/8563d73b96cc4f0a84dda30b6f1539b2] hfiles 2023-10-02T12:33:34,017 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/test-1696249987851/73dd4d167cdf4d422164d1f17ee13a19/f/8563d73b96cc4f0a84dda30b6f1539b2 for snapshot=snapshot_1696250013389_default_test-1696249987851 2023-10-02T12:33:34,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=45 2023-10-02T12:33:34,424 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1696249987851,,1696249988357.73dd4d167cdf4d422164d1f17ee13a19. 2023-10-02T12:33:34,425 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=46 2023-10-02T12:33:34,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.HMaster(3995): Remote procedure done, pid=46 2023-10-02T12:33:34,425 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696250013389_default_test-1696249987851 on region 73dd4d167cdf4d422164d1f17ee13a19 2023-10-02T12:33:34,427 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=46, ppid=45, state=RUNNABLE; SnapshotRegionProcedure 73dd4d167cdf4d422164d1f17ee13a19 2023-10-02T12:33:34,430 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=46, resume processing ppid=45 2023-10-02T12:33:34,430 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1696250013389_default_test-1696249987851 table=test-1696249987851 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:33:34,430 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=46, ppid=45, state=SUCCESS; SnapshotRegionProcedure 73dd4d167cdf4d422164d1f17ee13a19 in 566 msec 2023-10-02T12:33:34,431 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1696250013389_default_test-1696249987851 table=test-1696249987851 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:33:34,432 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1696250013389_default_test-1696249987851 table=test-1696249987851 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:33:34,432 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696250013389_default_test-1696249987851 2023-10-02T12:33:34,433 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/.hbase-snapshot/.tmp/snapshot_1696250013389_default_test-1696249987851 2023-10-02T12:33:34,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=45 2023-10-02T12:33:34,846 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1696250013389_default_test-1696249987851 table=test-1696249987851 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:33:34,853 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1696250013389_default_test-1696249987851 table=test-1696249987851 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:33:34,854 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/.hbase-snapshot/.tmp/snapshot_1696250013389_default_test-1696249987851 to hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/.hbase-snapshot/snapshot_1696250013389_default_test-1696249987851 2023-10-02T12:33:34,855 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1696250013389_default_test-1696249987851 table=test-1696249987851 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:33:34,855 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696250013389_default_test-1696249987851 table=test-1696249987851 type=FLUSH ttl=0 }, snapshot procedure id = 45 2023-10-02T12:33:34,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=45, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1696250013389_default_test-1696249987851 table=test-1696249987851 type=FLUSH ttl=0 } in 1.4310 sec 2023-10-02T12:33:35,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=45 2023-10-02T12:33:35,537 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: default:test-1696249987851, procId: 45 completed 2023-10-02T12:33:35,538 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1696250010959 2023-10-02T12:33:35,538 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-10-02T12:33:35,538 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-10-02T12:33:35,538 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1696250010959_test-1696249987851 2023-10-02T12:33:35,538 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1696250013389_default_test-1696249987851 to hdfs://localhost:46097/backupUT/backup_1696250010959/default/test-1696249987851/ 2023-10-02T12:33:35,538 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-10-02T12:33:35,562 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:45521, inputRoot=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560 2023-10-02T12:33:35,562 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,45341,1696249969200_1451628319_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:46097/backupUT/backup_1696250010959/default/test-1696249987851, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:46097/backupUT/backup_1696250010959/default/test-1696249987851/.hbase-snapshot/.tmp/snapshot_1696250013389_default_test-1696249987851 2023-10-02T12:33:35,562 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:45521, inputRoot=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560, snapshotDir=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/.hbase-snapshot/snapshot_1696250013389_default_test-1696249987851. 2023-10-02T12:33:35,575 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/.hbase-snapshot/snapshot_1696250013389_default_test-1696249987851 to hdfs://localhost:46097/backupUT/backup_1696250010959/default/test-1696249987851/.hbase-snapshot/.tmp/snapshot_1696250013389_default_test-1696249987851 2023-10-02T12:33:35,992 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:35,993 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:35,994 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:35,994 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:37,002 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/hadoop-2038350851834606204.jar 2023-10-02T12:33:37,003 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:37,003 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:37,004 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:37,004 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:37,005 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:37,005 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:37,005 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:37,006 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:37,006 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:37,007 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:33:37,008 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:33:37,008 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:33:37,009 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:33:37,009 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:33:37,010 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:33:37,010 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:33:37,011 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:33:37,011 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:33:37,012 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:33:37,013 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:33:37,013 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:33:37,014 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:33:37,015 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:33:37,016 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:33:37,016 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:33:37,016 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:33:37,017 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:33:37,017 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:33:37,046 WARN [Listener at jenkins-hbase3.apache.org/46241 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-10-02T12:33:37,070 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1696250013389_default_test-1696249987851' hfile list 2023-10-02T12:33:37,073 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-10-02T12:33:37,169 WARN [Listener at jenkins-hbase3.apache.org/46241 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/mapreduce.cluster.local.dir/1696250017126/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:33:37,170 WARN [Listener at jenkins-hbase3.apache.org/46241 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/mapreduce.cluster.local.dir/1696250017126/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:33:37,226 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-10-02T12:33:37,252 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/test-1696249987851/73dd4d167cdf4d422164d1f17ee13a19/f/8563d73b96cc4f0a84dda30b6f1539b2 output=hdfs://localhost:46097/backupUT/backup_1696250010959/default/test-1696249987851/archive/data/default/test-1696249987851/73dd4d167cdf4d422164d1f17ee13a19/f/8563d73b96cc4f0a84dda30b6f1539b2 2023-10-02T12:33:37,253 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 8.007M/sec 2023-10-02T12:33:37,440 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-10-02T12:33:37,440 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-10-02T12:33:38,209 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-10-02T12:33:38,210 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-10-02T12:33:38,218 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1696250013389_default_test-1696249987851 2023-10-02T12:33:38,219 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1696250013389_default_test-1696249987851 finished. 2023-10-02T12:33:38,219 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.BackupSystemTable(1034): test-1696249987851 2023-10-02T12:33:38,226 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696250010959 2023-10-02T12:33:38,226 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-10-02T12:33:38,635 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:46097/backupUT/backup_1696250010959/.backup.manifest 2023-10-02T12:33:38,635 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.TableBackupClient(358): Backup backup_1696250010959 finished: type=FULL,tablelist=test-1696249987851,targetRootDir=hdfs://localhost:46097/backupUT,startts=1696250013139,completets=1696250018225,bytescopied=0 2023-10-02T12:33:38,635 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-10-02T12:33:38,635 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1696250013389_default_test-1696249987851 2023-10-02T12:33:38,636 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696250013389_default_test-1696249987851" 2023-10-02T12:33:38,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696250013389_default_test-1696249987851 2023-10-02T12:33:38,640 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1696250013389_default_test-1696249987851 for backup backup_1696250010959 succeeded. 2023-10-02T12:33:38,640 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:33:38,646 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:33:38,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:33:38,650 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:33:38,652 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:33:38,660 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] impl.TableBackupClient(376): Backup backup_1696250010959 completed. 2023-10-02T12:33:38,660 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-02T12:33:38,660 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5a40feff to 127.0.0.1:49524 2023-10-02T12:33:38,660 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:33:38,660 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] backup.TestRemoteRestore(94): backup complete 2023-10-02T12:33:38,671 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/hadoop.log.dir so I do NOT create it in target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8 2023-10-02T12:33:38,672 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/hadoop.tmp.dir so I do NOT create it in target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8 2023-10-02T12:33:38,672 WARN [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.HBaseTestingUtility(455): hadoop.tmp.dir property value differs in configuration and system: Configuration=/tmp/hadoop-jenkins while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/hadoop.tmp.dir Erasing configuration value by system value. 2023-10-02T12:33:38,672 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8 2023-10-02T12:33:38,672 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/cluster_8d5394c9-0df2-fcee-025a-99149dac8b29, deleteOnExit=true 2023-10-02T12:33:38,672 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/test.cache.data in system properties and HBase conf 2023-10-02T12:33:38,672 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/hadoop.tmp.dir in system properties and HBase conf 2023-10-02T12:33:38,672 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/hadoop.log.dir in system properties and HBase conf 2023-10-02T12:33:38,672 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/mapreduce.cluster.local.dir in system properties and HBase conf 2023-10-02T12:33:38,672 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-10-02T12:33:38,672 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-10-02T12:33:38,673 DEBUG [Listener at jenkins-hbase3.apache.org/46241 {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-10-02T12:33:38,673 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-10-02T12:33:38,673 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-10-02T12:33:38,673 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-10-02T12:33:38,673 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-02T12:33:38,673 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-10-02T12:33:38,673 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-10-02T12:33:38,673 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-02T12:33:38,674 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-02T12:33:38,674 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-10-02T12:33:38,674 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/nfs.dump.dir in system properties and HBase conf 2023-10-02T12:33:38,674 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/java.io.tmpdir in system properties and HBase conf 2023-10-02T12:33:38,674 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-02T12:33:38,674 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-10-02T12:33:38,674 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-10-02T12:33:38,688 WARN [Listener at jenkins-hbase3.apache.org/46241 {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-10-02T12:33:38,688 WARN [Listener at jenkins-hbase3.apache.org/46241 {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-10-02T12:33:38,746 WARN [Listener at jenkins-hbase3.apache.org/46241 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:33:38,747 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:33:38,757 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/java.io.tmpdir/Jetty_localhost_46739_hdfs____.399id7/webapp 2023-10-02T12:33:38,864 INFO [Listener at jenkins-hbase3.apache.org/46241 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:46739 2023-10-02T12:33:38,877 WARN [Listener at jenkins-hbase3.apache.org/46241 {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-10-02T12:33:38,877 WARN [Listener at jenkins-hbase3.apache.org/46241 {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-10-02T12:33:38,941 WARN [Listener at localhost/35557 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-10-02T12:33:38,957 WARN [Listener at localhost/35557 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-10-02T12:33:38,959 WARN [Listener at localhost/35557 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:33:38,960 INFO [Listener at localhost/35557 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:33:38,967 INFO [Listener at localhost/35557 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/java.io.tmpdir/Jetty_localhost_35403_datanode____xpokjg/webapp 2023-10-02T12:33:39,068 INFO [Listener at localhost/35557 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:35403 2023-10-02T12:33:39,085 WARN [Listener at localhost/46345 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-10-02T12:33:39,102 WARN [Listener at localhost/46345 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-10-02T12:33:39,105 WARN [Listener at localhost/46345 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:33:39,106 INFO [Listener at localhost/46345 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:33:39,111 INFO [Listener at localhost/46345 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/java.io.tmpdir/Jetty_localhost_39649_datanode____.cr4qro/webapp 2023-10-02T12:33:39,172 WARN [Thread-1112 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-10-02T12:33:39,210 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xdf409bfee46f54ae: Processing first storage report for DS-072f6d50-0563-4e17-90ee-68e580f0209f from datanode bedd33f4-c125-4c79-b729-d9696980602c 2023-10-02T12:33:39,211 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xdf409bfee46f54ae: from storage DS-072f6d50-0563-4e17-90ee-68e580f0209f node DatanodeRegistration(127.0.0.1:34747, datanodeUuid=bedd33f4-c125-4c79-b729-d9696980602c, infoPort=43429, infoSecurePort=0, ipcPort=46345, storageInfo=lv=-57;cid=testClusterID;nsid=1078312054;c=1696250018691), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2023-10-02T12:33:39,211 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xdf409bfee46f54ae: Processing first storage report for DS-d4e360b7-6ff6-4872-a0d1-d8e1633e6aa9 from datanode bedd33f4-c125-4c79-b729-d9696980602c 2023-10-02T12:33:39,211 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xdf409bfee46f54ae: from storage DS-d4e360b7-6ff6-4872-a0d1-d8e1633e6aa9 node DatanodeRegistration(127.0.0.1:34747, datanodeUuid=bedd33f4-c125-4c79-b729-d9696980602c, infoPort=43429, infoSecurePort=0, ipcPort=46345, storageInfo=lv=-57;cid=testClusterID;nsid=1078312054;c=1696250018691), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-10-02T12:33:39,233 INFO [Listener at localhost/46345 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:39649 2023-10-02T12:33:39,255 WARN [Listener at localhost/42035 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-10-02T12:33:39,321 WARN [Listener at localhost/42035 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-10-02T12:33:39,323 WARN [Listener at localhost/42035 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:33:39,325 INFO [Listener at localhost/42035 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:33:39,334 INFO [Listener at localhost/42035 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/java.io.tmpdir/Jetty_localhost_38473_datanode____kvxxpu/webapp 2023-10-02T12:33:39,349 WARN [Thread-1147 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-10-02T12:33:39,386 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x30adda72c1791d2f: Processing first storage report for DS-7d2573b0-e090-426d-a70f-2e6b5dc3b6e7 from datanode 346eac38-a465-40a3-8b5b-394b0f7dd334 2023-10-02T12:33:39,386 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x30adda72c1791d2f: from storage DS-7d2573b0-e090-426d-a70f-2e6b5dc3b6e7 node DatanodeRegistration(127.0.0.1:33235, datanodeUuid=346eac38-a465-40a3-8b5b-394b0f7dd334, infoPort=44223, infoSecurePort=0, ipcPort=42035, storageInfo=lv=-57;cid=testClusterID;nsid=1078312054;c=1696250018691), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2023-10-02T12:33:39,386 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x30adda72c1791d2f: Processing first storage report for DS-62775615-e0f4-4d0e-8a49-cdfb4a6e48b6 from datanode 346eac38-a465-40a3-8b5b-394b0f7dd334 2023-10-02T12:33:39,386 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x30adda72c1791d2f: from storage DS-62775615-e0f4-4d0e-8a49-cdfb4a6e48b6 node DatanodeRegistration(127.0.0.1:33235, datanodeUuid=346eac38-a465-40a3-8b5b-394b0f7dd334, infoPort=44223, infoSecurePort=0, ipcPort=42035, storageInfo=lv=-57;cid=testClusterID;nsid=1078312054;c=1696250018691), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-10-02T12:33:39,476 INFO [Listener at localhost/42035 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:38473 2023-10-02T12:33:39,497 WARN [Listener at localhost/37283 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-10-02T12:33:39,599 WARN [Thread-1181 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-10-02T12:33:39,640 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x1f0b4c7480bd08ea: Processing first storage report for DS-5ead604a-09a4-4a6e-b18c-784c76921905 from datanode 5170f853-bb77-4d7f-b201-9315f8ce902a 2023-10-02T12:33:39,640 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x1f0b4c7480bd08ea: from storage DS-5ead604a-09a4-4a6e-b18c-784c76921905 node DatanodeRegistration(127.0.0.1:37913, datanodeUuid=5170f853-bb77-4d7f-b201-9315f8ce902a, infoPort=34913, infoSecurePort=0, ipcPort=37283, storageInfo=lv=-57;cid=testClusterID;nsid=1078312054;c=1696250018691), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2023-10-02T12:33:39,640 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x1f0b4c7480bd08ea: Processing first storage report for DS-1f594269-ca98-4f24-a582-5dc5c18b7663 from datanode 5170f853-bb77-4d7f-b201-9315f8ce902a 2023-10-02T12:33:39,640 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x1f0b4c7480bd08ea: from storage DS-1f594269-ca98-4f24-a582-5dc5c18b7663 node DatanodeRegistration(127.0.0.1:37913, datanodeUuid=5170f853-bb77-4d7f-b201-9315f8ce902a, infoPort=34913, infoSecurePort=0, ipcPort=37283, storageInfo=lv=-57;cid=testClusterID;nsid=1078312054;c=1696250018691), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-10-02T12:33:39,714 DEBUG [Listener at localhost/37283 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8 2023-10-02T12:33:39,714 INFO [Listener at localhost/37283 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-10-02T12:33:39,714 INFO [Listener at localhost/37283 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/test.cache.data in system properties and HBase conf 2023-10-02T12:33:39,714 INFO [Listener at localhost/37283 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/hadoop.tmp.dir in system properties and HBase conf 2023-10-02T12:33:39,715 INFO [Listener at localhost/37283 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/hadoop.log.dir in system properties and HBase conf 2023-10-02T12:33:39,715 INFO [Listener at localhost/37283 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/mapreduce.cluster.local.dir in system properties and HBase conf 2023-10-02T12:33:39,715 INFO [Listener at localhost/37283 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-10-02T12:33:39,715 INFO [Listener at localhost/37283 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-10-02T12:33:39,715 INFO [Listener at localhost/37283 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:33:39,717 INFO [Listener at localhost/37283 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:33:39,722 INFO [Listener at localhost/37283 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-10-02T12:33:39,722 INFO [Listener at localhost/37283 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-10-02T12:33:39,722 INFO [Listener at localhost/37283 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-10-02T12:33:39,722 INFO [Listener at localhost/37283 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-02T12:33:39,722 INFO [Listener at localhost/37283 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-10-02T12:33:39,722 INFO [Listener at localhost/37283 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-10-02T12:33:39,723 INFO [Listener at localhost/37283 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-02T12:33:39,723 INFO [Listener at localhost/37283 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-02T12:33:39,723 INFO [Listener at localhost/37283 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-10-02T12:33:39,723 INFO [Listener at localhost/37283 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/nfs.dump.dir in system properties and HBase conf 2023-10-02T12:33:39,723 INFO [Listener at localhost/37283 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/java.io.tmpdir in system properties and HBase conf 2023-10-02T12:33:39,723 INFO [Listener at localhost/37283 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-02T12:33:39,723 INFO [Listener at localhost/37283 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-10-02T12:33:39,723 INFO [Listener at localhost/37283 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-10-02T12:33:40,342 WARN [Thread-1300 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:33:40,351 WARN [Thread-1300 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-10-02T12:33:40,351 INFO [Thread-1300 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:33:40,354 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2023-10-02T12:33:40,361 INFO [Thread-1300 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_42041_jobhistory____.q750uf/webapp 2023-10-02T12:33:40,426 INFO [Thread-1300 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-10-02T12:33:40,426 INFO [Thread-1300 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-10-02T12:33:40,426 INFO [Thread-1300 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-10-02T12:33:40,427 INFO [Thread-1300 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-10-02T12:33:40,436 INFO [Thread-1300 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:33:40,497 INFO [Thread-1300 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:33:40,584 INFO [Thread-1300 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-10-02T12:33:40,594 INFO [Thread-1300 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:42041 2023-10-02T12:33:41,828 WARN [Listener at jenkins-hbase3.apache.org/39355 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:33:41,839 WARN [Listener at jenkins-hbase3.apache.org/39355 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-10-02T12:33:41,839 INFO [Listener at jenkins-hbase3.apache.org/39355 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:33:41,847 INFO [Listener at jenkins-hbase3.apache.org/39355 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_41143_cluster____tz7asw/webapp 2023-10-02T12:33:41,988 INFO [Listener at jenkins-hbase3.apache.org/39355 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-10-02T12:33:41,988 INFO [Listener at jenkins-hbase3.apache.org/39355 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-10-02T12:33:41,988 INFO [Listener at jenkins-hbase3.apache.org/39355 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-10-02T12:33:41,989 INFO [Listener at jenkins-hbase3.apache.org/39355 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-10-02T12:33:41,998 INFO [Listener at jenkins-hbase3.apache.org/39355 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:33:42,130 INFO [Listener at jenkins-hbase3.apache.org/39355 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:33:42,217 INFO [Listener at jenkins-hbase3.apache.org/39355 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:33:42,228 INFO [Listener at jenkins-hbase3.apache.org/39355 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:41143 2023-10-02T12:33:42,393 WARN [Listener at jenkins-hbase3.apache.org/39189 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:33:42,399 WARN [Listener at jenkins-hbase3.apache.org/39189 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-10-02T12:33:42,400 INFO [Listener at jenkins-hbase3.apache.org/39189 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:33:42,407 INFO [Listener at jenkins-hbase3.apache.org/39189 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_46087_node____.y1fgc4/webapp 2023-10-02T12:33:42,467 INFO [Listener at jenkins-hbase3.apache.org/39189 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-10-02T12:33:42,467 INFO [Listener at jenkins-hbase3.apache.org/39189 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-10-02T12:33:42,468 INFO [Listener at jenkins-hbase3.apache.org/39189 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-10-02T12:33:42,468 INFO [Listener at jenkins-hbase3.apache.org/39189 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-10-02T12:33:42,475 INFO [Listener at jenkins-hbase3.apache.org/39189 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:33:42,494 INFO [Listener at jenkins-hbase3.apache.org/39189 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:33:42,564 INFO [Listener at jenkins-hbase3.apache.org/39189 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:33:42,580 INFO [Listener at jenkins-hbase3.apache.org/39189 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:46087 2023-10-02T12:33:42,770 WARN [Listener at jenkins-hbase3.apache.org/43893 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:33:42,778 WARN [Listener at jenkins-hbase3.apache.org/43893 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-10-02T12:33:42,779 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:33:42,799 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_34007_node____.7gggiz/webapp 2023-10-02T12:33:42,907 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-10-02T12:33:42,907 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-10-02T12:33:42,908 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-10-02T12:33:42,908 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-10-02T12:33:42,918 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:33:42,941 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:33:42,957 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-10-02T12:33:42,957 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-10-02T12:33:43,030 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:33:43,039 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:34007 2023-10-02T12:33:43,045 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-10-02T12:33:43,047 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5691222f to 127.0.0.1:49524 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:33:43,054 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ca6559, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:33:43,056 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:33:43,059 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:59512, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:33:43,061 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] impl.RestoreTablesClient(101): HBase table table1 does not exist. It will be created during restore process 2023-10-02T12:33:43,062 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:46097/backupUT/backup_1696250010959/.backup.manifest 2023-10-02T12:33:43,065 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250010959/.backup.manifest 2023-10-02T12:33:43,066 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:46097/backupUT/backup_1696250010959/.backup.manifest 2023-10-02T12:33:43,069 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250010959/.backup.manifest 2023-10-02T12:33:43,069 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] impl.RestoreTablesClient(148): Restoring 'test-1696249987851' to 'table1' from full backup image hdfs://localhost:46097/backupUT/backup_1696250010959/default/test-1696249987851 2023-10-02T12:33:43,083 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] util.RestoreTool(487): Creating target table 'table1' 2023-10-02T12:33:43,083 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:46097/backupUT/backup_1696250010959/default/test-1696249987851/archive/data/default/test-1696249987851/73dd4d167cdf4d422164d1f17ee13a19 2023-10-02T12:33:43,084 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:46097/backupUT/backup_1696250010959/default/test-1696249987851/archive/data/default/test-1696249987851/73dd4d167cdf4d422164d1f17ee13a19/f in region [hdfs://localhost:46097/backupUT/backup_1696250010959/default/test-1696249987851/archive/data/default/test-1696249987851/73dd4d167cdf4d422164d1f17ee13a19] 2023-10-02T12:33:43,088 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] util.RestoreTool(442): Trying to figure out region boundaries hfile=hdfs://localhost:46097/backupUT/backup_1696250010959/default/test-1696249987851/archive/data/default/test-1696249987851/73dd4d167cdf4d422164d1f17ee13a19/f/8563d73b96cc4f0a84dda30b6f1539b2 first=row0 last=row98 2023-10-02T12:33:43,089 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:33:43,095 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:38342, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:33:43,097 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:33:43,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] procedure2.ProcedureExecutor(1032): Stored pid=47, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=table1 2023-10-02T12:33:43,103 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=47, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:33:43,103 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 47 2023-10-02T12:33:43,104 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=47, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:33:43,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-10-02T12:33:43,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-10-02T12:33:43,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-10-02T12:33:43,515 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 7413dbaac031709c0a046de1e26ef975, NAME => 'table1,,1696250023097.7413dbaac031709c0a046de1e26ef975.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560 2023-10-02T12:33:43,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-10-02T12:33:43,922 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(888): Instantiated table1,,1696250023097.7413dbaac031709c0a046de1e26ef975.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:43,922 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1668): Closing 7413dbaac031709c0a046de1e26ef975, disabling compactions & flushes 2023-10-02T12:33:43,922 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1690): Closing region table1,,1696250023097.7413dbaac031709c0a046de1e26ef975. 2023-10-02T12:33:43,922 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1696250023097.7413dbaac031709c0a046de1e26ef975. 2023-10-02T12:33:43,922 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on table1,,1696250023097.7413dbaac031709c0a046de1e26ef975. after waiting 0 ms 2023-10-02T12:33:43,922 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region table1,,1696250023097.7413dbaac031709c0a046de1e26ef975. 2023-10-02T12:33:43,922 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1909): Closed table1,,1696250023097.7413dbaac031709c0a046de1e26ef975. 2023-10-02T12:33:43,922 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1622): Region close journal for 7413dbaac031709c0a046de1e26ef975: 2023-10-02T12:33:43,924 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=47, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:33:43,924 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"table1,,1696250023097.7413dbaac031709c0a046de1e26ef975.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1696250023924"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696250023924"}]},"ts":"1696250023924"} 2023-10-02T12:33:43,925 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:33:43,926 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=47, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:33:43,926 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250023926"}]},"ts":"1696250023926"} 2023-10-02T12:33:43,928 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLING in hbase:meta 2023-10-02T12:33:43,931 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=7413dbaac031709c0a046de1e26ef975, ASSIGN}] 2023-10-02T12:33:43,934 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=7413dbaac031709c0a046de1e26ef975, ASSIGN 2023-10-02T12:33:43,935 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=table1, region=7413dbaac031709c0a046de1e26ef975, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38375,1696249972520; forceNewPlan=false, retain=false 2023-10-02T12:33:44,088 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=48 updating hbase:meta row=7413dbaac031709c0a046de1e26ef975, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:44,090 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE; OpenRegionProcedure 7413dbaac031709c0a046de1e26ef975, server=jenkins-hbase3.apache.org,38375,1696249972520}] 2023-10-02T12:33:44,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-10-02T12:33:44,246 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(130): Open table1,,1696250023097.7413dbaac031709c0a046de1e26ef975. 2023-10-02T12:33:44,246 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7254): Opening region: {ENCODED => 7413dbaac031709c0a046de1e26ef975, NAME => 'table1,,1696250023097.7413dbaac031709c0a046de1e26ef975.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:33:44,246 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:33:44,246 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 7413dbaac031709c0a046de1e26ef975 2023-10-02T12:33:44,246 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(888): Instantiated table1,,1696250023097.7413dbaac031709c0a046de1e26ef975.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:44,246 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7296): checking encryption for 7413dbaac031709c0a046de1e26ef975 2023-10-02T12:33:44,247 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7299): checking classloading for 7413dbaac031709c0a046de1e26ef975 2023-10-02T12:33:44,248 INFO [StoreOpener-7413dbaac031709c0a046de1e26ef975-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 7413dbaac031709c0a046de1e26ef975 2023-10-02T12:33:44,249 INFO [StoreOpener-7413dbaac031709c0a046de1e26ef975-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7413dbaac031709c0a046de1e26ef975 columnFamilyName f 2023-10-02T12:33:44,249 DEBUG [StoreOpener-7413dbaac031709c0a046de1e26ef975-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:44,250 INFO [StoreOpener-7413dbaac031709c0a046de1e26ef975-1 {}] regionserver.HStore(324): Store=7413dbaac031709c0a046de1e26ef975/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:33:44,250 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/table1/7413dbaac031709c0a046de1e26ef975 2023-10-02T12:33:44,251 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/table1/7413dbaac031709c0a046de1e26ef975 2023-10-02T12:33:44,254 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1076): writing seq id for 7413dbaac031709c0a046de1e26ef975 2023-10-02T12:33:44,256 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/table1/7413dbaac031709c0a046de1e26ef975/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:33:44,256 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): Opened 7413dbaac031709c0a046de1e26ef975; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10663432000, jitterRate=-0.00689050555229187}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:33:44,257 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(992): Region open journal for 7413dbaac031709c0a046de1e26ef975: 2023-10-02T12:33:44,258 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2556): Post open deploy tasks for table1,,1696250023097.7413dbaac031709c0a046de1e26ef975., pid=49, masterSystemTime=1696250024242 2023-10-02T12:33:44,259 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2583): Finished post open deploy task for table1,,1696250023097.7413dbaac031709c0a046de1e26ef975. 2023-10-02T12:33:44,260 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(158): Opened table1,,1696250023097.7413dbaac031709c0a046de1e26ef975. 2023-10-02T12:33:44,260 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=48 updating hbase:meta row=7413dbaac031709c0a046de1e26ef975, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:33:44,264 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=49, resume processing ppid=48 2023-10-02T12:33:44,264 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=49, ppid=48, state=SUCCESS; OpenRegionProcedure 7413dbaac031709c0a046de1e26ef975, server=jenkins-hbase3.apache.org,38375,1696249972520 in 172 msec 2023-10-02T12:33:44,266 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=48, resume processing ppid=47 2023-10-02T12:33:44,266 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=48, ppid=47, state=SUCCESS; TransitRegionStateProcedure table=table1, region=7413dbaac031709c0a046de1e26ef975, ASSIGN in 333 msec 2023-10-02T12:33:44,266 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=47, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:33:44,267 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250024266"}]},"ts":"1696250024266"} 2023-10-02T12:33:44,268 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLED in hbase:meta 2023-10-02T12:33:44,272 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=47, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:33:44,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=47, state=SUCCESS; CreateTableProcedure table=table1 in 1.1750 sec 2023-10-02T12:33:45,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-10-02T12:33:45,216 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:table1, procId: 47 completed 2023-10-02T12:33:45,226 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:46097/backupUT/backup_1696250010959/default/test-1696249987851/archive/data/default/test-1696249987851/73dd4d167cdf4d422164d1f17ee13a19 from hbase tables test-1696249987851 to tables table1 2023-10-02T12:33:45,226 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.MapReduceRestoreJob(72): Restore test-1696249987851 into table1 2023-10-02T12:33:45,228 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:45521/backupUT/bulk_output-default-table1-1696250025226 from hdfs://localhost:46097/backupUT/backup_1696250010959/default/test-1696249987851/archive/data/default/test-1696249987851/73dd4d167cdf4d422164d1f17ee13a19 2023-10-02T12:33:45,229 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3e379db7 to 127.0.0.1:49524 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:33:45,236 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@747c95b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:33:45,238 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:33:45,240 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:38346, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:33:45,241 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-10-02T12:33:45,241 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table table1 2023-10-02T12:33:45,248 DEBUG [hconnection-0x5c076283-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:33:45,250 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:59520, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:33:45,252 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for table1: 2023-10-02T12:33:45,252 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-10-02T12:33:45,253 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to /user/jenkins/hbase-staging/partitions_55339a73-1b81-485a-af41-97541873dea6 2023-10-02T12:33:45,263 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:45,263 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:45,264 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:45,264 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:46,141 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:33:46,290 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/hadoop-638400510762715828.jar 2023-10-02T12:33:46,291 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:46,291 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:46,292 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:46,292 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:46,293 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:46,293 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:46,293 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:46,294 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:46,294 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:46,295 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:33:46,295 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:33:46,296 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:33:46,296 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:33:46,297 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:33:46,297 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:33:46,297 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:33:46,298 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:33:46,298 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:33:46,299 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:33:46,299 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:33:46,299 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:33:46,300 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:46,301 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:46,301 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:46,301 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:46,301 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:46,302 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:46,303 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:33:46,303 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: table1 2023-10-02T12:33:46,303 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-02T12:33:46,311 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3e379db7 to 127.0.0.1:49524 2023-10-02T12:33:46,311 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:33:46,311 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-10-02T12:33:46,311 WARN [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-10-02T12:33:46,312 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:33:46,571 WARN [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-10-02T12:33:47,152 WARN [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-10-02T12:33:47,624 WARN [SchedulerEventDispatcher:Event Processor {}] capacity.LeafQueue(815): maximum-am-resource-percent is insufficient to start a single application in queue, it is likely set too low. skipping enforcement to allow at least one application to start 2023-10-02T12:33:47,624 WARN [SchedulerEventDispatcher:Event Processor {}] capacity.LeafQueue(851): maximum-am-resource-percent is insufficient to start a single application in queue for user, it is likely set too low. skipping enforcement to allow at least one application to start 2023-10-02T12:33:48,242 INFO [Socket Reader #1 for port 0 {}] ipc.Server$Connection(2001): Auth successful for appattempt_1696250022245_0001_000001 (auth:SIMPLE) 2023-10-02T12:33:49,313 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'table1' 2023-10-02T12:33:50,354 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2023-10-02T12:33:50,354 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_table1 Metrics about Tables on a single HBase RegionServer 2023-10-02T12:33:53,811 INFO [Socket Reader #1 for port 0 {}] ipc.Server$Connection(2001): Auth successful for appattempt_1696250022245_0001_000001 (auth:SIMPLE) 2023-10-02T12:33:56,100 INFO [Socket Reader #1 for port 0 {}] ipc.Server$Connection(2001): Auth successful for appattempt_1696250022245_0001_000001 (auth:SIMPLE) 2023-10-02T12:33:56,689 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region df86f7708dbdd71bc4105e1e70256952 changed from -1.0 to 0.0, refreshing cache 2023-10-02T12:34:01,588 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2023-10-02T12:34:01,588 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region c28d048d5108c73bc091dea0057bf483 changed from -1.0 to 0.0, refreshing cache 2023-10-02T12:34:01,974 INFO [Socket Reader #1 for port 0 {}] ipc.Server$Connection(2001): Auth successful for appattempt_1696250022245_0001_000001 (auth:SIMPLE) 2023-10-02T12:34:06,225 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45268, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:34:06,650 INFO [Socket Reader #1 for port 0 {}] ipc.Server$Connection(2001): Auth successful for appattempt_1696250022245_0001_000001 (auth:SIMPLE) 2023-10-02T12:34:06,668 WARN [ContainersLauncher #1 {}] nodemanager.DefaultContainerExecutor(305): Exit code from container container_1696250022245_0001_01_000003 is : 143 2023-10-02T12:34:08,043 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:45521/backupUT/bulk_output-default-table1-1696250025226 2023-10-02T12:34:08,044 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x72891370 to 127.0.0.1:49524 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:34:08,055 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cfa4003, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:34:08,057 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:34:08,062 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45270, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:34:08,081 WARN [Listener at jenkins-hbase3.apache.org/43893 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:45521/backupUT/bulk_output-default-table1-1696250025226/_SUCCESS 2023-10-02T12:34:08,088 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:34:08,094 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:35076, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:34:08,117 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:45521/backupUT/bulk_output-default-table1-1696250025226/f/32be6d43ddab4d669ec6c58e31dabd83 first=Optional[row0] last=Optional[row98] 2023-10-02T12:34:08,119 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=table1,,1696250023097.7413dbaac031709c0a046de1e26ef975., hostname=jenkins-hbase3.apache.org,38375,1696249972520, seqNum=2 for row with hfile group [{f,hdfs://localhost:45521/backupUT/bulk_output-default-table1-1696250025226/f/32be6d43ddab4d669ec6c58e31dabd83}] 2023-10-02T12:34:08,129 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:45521/backupUT/bulk_output-default-table1-1696250025226/f/32be6d43ddab4d669ec6c58e31dabd83 for inclusion in 7413dbaac031709c0a046de1e26ef975/f 2023-10-02T12:34:08,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-10-02T12:34:08,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] regionserver.HStore(625): Region bounds: first= last= 2023-10-02T12:34:08,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] regionserver.HRegion(2520): Flush status journal for 7413dbaac031709c0a046de1e26ef975: 2023-10-02T12:34:08,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:45521/backupUT/bulk_output-default-table1-1696250025226/f/32be6d43ddab4d669ec6c58e31dabd83 to hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/staging/jenkins__table1__flao913j4vfkbrd7mee6qfcod82aa7pe27t9qr0tg5ou2ojjofe9ndlrfd8q9git/f/32be6d43ddab4d669ec6c58e31dabd83 2023-10-02T12:34:08,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/staging/jenkins__table1__flao913j4vfkbrd7mee6qfcod82aa7pe27t9qr0tg5ou2ojjofe9ndlrfd8q9git/f/32be6d43ddab4d669ec6c58e31dabd83 as hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/table1/7413dbaac031709c0a046de1e26ef975/f/abaf51e1055d40588de74889223c843e_SeqId_4_ 2023-10-02T12:34:08,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x49c498da to 127.0.0.1:49524 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:34:08,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2463ad4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:34:08,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:34:08,155 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:35082, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:34:08,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:34:08,160 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45278, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:34:08,179 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-02T12:34:08,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x49c498da to 127.0.0.1:49524 2023-10-02T12:34:08,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:08,181 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/staging/jenkins__table1__flao913j4vfkbrd7mee6qfcod82aa7pe27t9qr0tg5ou2ojjofe9ndlrfd8q9git/f/32be6d43ddab4d669ec6c58e31dabd83 into 7413dbaac031709c0a046de1e26ef975/f as hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/table1/7413dbaac031709c0a046de1e26ef975/f/abaf51e1055d40588de74889223c843e_SeqId_4_ - updating store file list. 2023-10-02T12:34:08,188 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/table1/7413dbaac031709c0a046de1e26ef975/f/abaf51e1055d40588de74889223c843e_SeqId_4_ into 7413dbaac031709c0a046de1e26ef975/f 2023-10-02T12:34:08,188 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/staging/jenkins__table1__flao913j4vfkbrd7mee6qfcod82aa7pe27t9qr0tg5ou2ojjofe9ndlrfd8q9git/f/32be6d43ddab4d669ec6c58e31dabd83 into 7413dbaac031709c0a046de1e26ef975/f (new location: hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/table1/7413dbaac031709c0a046de1e26ef975/f/abaf51e1055d40588de74889223c843e_SeqId_4_) 2023-10-02T12:34:08,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/staging/jenkins__table1__flao913j4vfkbrd7mee6qfcod82aa7pe27t9qr0tg5ou2ojjofe9ndlrfd8q9git/f/32be6d43ddab4d669ec6c58e31dabd83 2023-10-02T12:34:08,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4594a031 to 127.0.0.1:49524 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:34:08,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c625f44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:34:08,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:34:08,203 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:35086, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-10-02T12:34:08,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:34:08,211 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45292, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-10-02T12:34:08,238 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-02T12:34:08,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x4594a031 to 127.0.0.1:49524 2023-10-02T12:34:08,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:08,241 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-02T12:34:08,242 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x72891370 to 127.0.0.1:49524 2023-10-02T12:34:08,242 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:08,242 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-10-02T12:34:08,242 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-10-02T12:34:08,242 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] impl.RestoreTablesClient(233): Backup: backup_1696250010959 hdfs://localhost:46097/backupUT/backup_1696250010959/default/test-1696249987851/ 2023-10-02T12:34:08,242 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-10-02T12:34:08,260 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] client.HBaseAdmin$18(967): Started disable of table1 2023-10-02T12:34:08,260 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable table1 2023-10-02T12:34:08,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] procedure2.ProcedureExecutor(1032): Stored pid=50, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=table1 2023-10-02T12:34:08,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-10-02T12:34:08,267 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250048267"}]},"ts":"1696250048267"} 2023-10-02T12:34:08,270 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLING in hbase:meta 2023-10-02T12:34:08,277 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(293): Set table1 to state=DISABLING 2023-10-02T12:34:08,278 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=7413dbaac031709c0a046de1e26ef975, UNASSIGN}] 2023-10-02T12:34:08,281 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=51, ppid=50, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=7413dbaac031709c0a046de1e26ef975, UNASSIGN 2023-10-02T12:34:08,282 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=51 updating hbase:meta row=7413dbaac031709c0a046de1e26ef975, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:34:08,284 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE; CloseRegionProcedure 7413dbaac031709c0a046de1e26ef975, server=jenkins-hbase3.apache.org,38375,1696249972520}] 2023-10-02T12:34:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-10-02T12:34:08,436 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] handler.UnassignRegionHandler(119): Close 7413dbaac031709c0a046de1e26ef975 2023-10-02T12:34:08,437 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1668): Closing 7413dbaac031709c0a046de1e26ef975, disabling compactions & flushes 2023-10-02T12:34:08,437 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1690): Closing region table1,,1696250023097.7413dbaac031709c0a046de1e26ef975. 2023-10-02T12:34:08,437 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1696250023097.7413dbaac031709c0a046de1e26ef975. 2023-10-02T12:34:08,437 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1778): Acquired close lock on table1,,1696250023097.7413dbaac031709c0a046de1e26ef975. after waiting 0 ms 2023-10-02T12:34:08,437 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1788): Updates disabled for region table1,,1696250023097.7413dbaac031709c0a046de1e26ef975. 2023-10-02T12:34:08,448 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/table1/7413dbaac031709c0a046de1e26ef975/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-10-02T12:34:08,449 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:34:08,451 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1909): Closed table1,,1696250023097.7413dbaac031709c0a046de1e26ef975. 2023-10-02T12:34:08,451 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1622): Region close journal for 7413dbaac031709c0a046de1e26ef975: 2023-10-02T12:34:08,453 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] handler.UnassignRegionHandler(163): Closed 7413dbaac031709c0a046de1e26ef975 2023-10-02T12:34:08,454 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=51 updating hbase:meta row=7413dbaac031709c0a046de1e26ef975, regionState=CLOSED 2023-10-02T12:34:08,458 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=52, resume processing ppid=51 2023-10-02T12:34:08,458 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=52, ppid=51, state=SUCCESS; CloseRegionProcedure 7413dbaac031709c0a046de1e26ef975, server=jenkins-hbase3.apache.org,38375,1696249972520 in 172 msec 2023-10-02T12:34:08,460 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=51, resume processing ppid=50 2023-10-02T12:34:08,460 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=51, ppid=50, state=SUCCESS; TransitRegionStateProcedure table=table1, region=7413dbaac031709c0a046de1e26ef975, UNASSIGN in 180 msec 2023-10-02T12:34:08,461 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250048460"}]},"ts":"1696250048460"} 2023-10-02T12:34:08,462 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLED in hbase:meta 2023-10-02T12:34:08,464 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(305): Set table1 to state=DISABLED 2023-10-02T12:34:08,466 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=50, state=SUCCESS; DisableTableProcedure table=table1 in 205 msec 2023-10-02T12:34:08,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-10-02T12:34:08,570 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] client.HBaseAdmin$TableFuture(3677): Operation: DISABLE, Table Name: default:table1, procId: 50 completed 2023-10-02T12:34:08,571 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.HMaster$5(2448): Client=jenkins//172.31.12.81 delete table1 2023-10-02T12:34:08,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] procedure2.ProcedureExecutor(1032): Stored pid=53, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=table1 2023-10-02T12:34:08,576 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=53, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-10-02T12:34:08,577 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=53, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=table1 2023-10-02T12:34:08,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=53 2023-10-02T12:34:08,588 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/table1/7413dbaac031709c0a046de1e26ef975 2023-10-02T12:34:08,596 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/table1/7413dbaac031709c0a046de1e26ef975/f, FileablePath, hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/table1/7413dbaac031709c0a046de1e26ef975/recovered.edits] 2023-10-02T12:34:08,603 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/table1/7413dbaac031709c0a046de1e26ef975/f/abaf51e1055d40588de74889223c843e_SeqId_4_ to hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/archive/data/default/table1/7413dbaac031709c0a046de1e26ef975/f/abaf51e1055d40588de74889223c843e_SeqId_4_ 2023-10-02T12:34:08,608 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/table1/7413dbaac031709c0a046de1e26ef975/recovered.edits/6.seqid to hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/archive/data/default/table1/7413dbaac031709c0a046de1e26ef975/recovered.edits/6.seqid 2023-10-02T12:34:08,611 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/table1/7413dbaac031709c0a046de1e26ef975 2023-10-02T12:34:08,611 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(293): Archived table1 regions 2023-10-02T12:34:08,615 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=53, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=table1 2023-10-02T12:34:08,620 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of table1 from hbase:meta 2023-10-02T12:34:08,633 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(387): Removing 'table1' descriptor. 2023-10-02T12:34:08,635 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=53, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=table1 2023-10-02T12:34:08,635 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(377): Removing 'table1' from region states. 2023-10-02T12:34:08,635 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1,,1696250023097.7413dbaac031709c0a046de1e26ef975.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1696250048635"}]},"ts":"9223372036854775807"} 2023-10-02T12:34:08,640 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-10-02T12:34:08,640 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 7413dbaac031709c0a046de1e26ef975, NAME => 'table1,,1696250023097.7413dbaac031709c0a046de1e26ef975.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:34:08,640 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(381): Marking 'table1' as deleted. 2023-10-02T12:34:08,640 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1696250048640"}]},"ts":"9223372036854775807"} 2023-10-02T12:34:08,654 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1678): Deleted table table1 state from META 2023-10-02T12:34:08,659 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(127): Finished pid=53, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-10-02T12:34:08,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=53, state=SUCCESS; DeleteTableProcedure table=table1 in 88 msec 2023-10-02T12:34:08,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45341 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=53 2023-10-02T12:34:08,679 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] client.HBaseAdmin$TableFuture(3677): Operation: DELETE, Table Name: default:table1, procId: 53 completed 2023-10-02T12:34:08,768 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] hbase.ResourceChecker(175): after: backup.TestRemoteRestore#testFullRestoreRemoteWithAlternateRestoreOutputDir Thread=1305 (was 830) Potentially hanging thread: IPC Server handler 9 on default port 46763 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 1994747619@qtp-192977947-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:39649 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: New I/O worker #57 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 0 on default port 43893 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 29 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: DataNode DiskChecker thread 1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 49 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-56 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 0 on default port 39201 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #56 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x46f0727e-shared-pool-12 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 18 on default port 46763 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-400-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 2 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 37 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-45 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 38 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 9 on default port 39201 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 13 on default port 39201 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 7 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 4 on default port 39189 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 33 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Listener at jenkins-hbase3.apache.org/43893 java.lang.Thread.dumpThreads(Native Method) java.lang.Thread.getAllStackTraces(Thread.java:1615) org.apache.hadoop.hbase.ResourceCheckerJUnitListener$ThreadResourceAnalyzer.getVal(ResourceCheckerJUnitListener.java:49) org.apache.hadoop.hbase.ResourceChecker.fill(ResourceChecker.java:110) org.apache.hadoop.hbase.ResourceChecker.fillEndings(ResourceChecker.java:104) org.apache.hadoop.hbase.ResourceChecker.end(ResourceChecker.java:206) org.apache.hadoop.hbase.ResourceCheckerJUnitListener.end(ResourceCheckerJUnitListener.java:165) org.apache.hadoop.hbase.ResourceCheckerJUnitListener.testFinished(ResourceCheckerJUnitListener.java:185) org.junit.runner.notification.SynchronizedRunListener.testFinished(SynchronizedRunListener.java:87) org.junit.runner.notification.RunNotifier$9.notifyListener(RunNotifier.java:225) org.junit.runner.notification.RunNotifier$SafeNotifier.run(RunNotifier.java:72) org.junit.runner.notification.RunNotifier.fireTestFinished(RunNotifier.java:222) org.junit.internal.runners.model.EachTestNotifier.fireTestFinished(EachTestNotifier.java:38) org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:372) org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) java.util.concurrent.FutureTask.run(FutureTask.java:266) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 0 on default port 46763 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: hconnection-0x46f0727e-shared-pool-13 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: 1567292688@qtp-263136829-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:42041 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: IPC Server handler 3 on default port 35557 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 41 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 4 on default port 43893 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 7 on default port 44597 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 41 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 39189 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 7 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 27 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-53 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 4 on default port 35557 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-41 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 4 on default port 46345 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: hconnection-0x46f0727e-shared-pool-15 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 5 on default port 46763 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 32 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 20 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 12 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 5 on default port 44597 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 32 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 18 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeEditLogRoller@229975 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeEditLogRoller.run(FSNamesystem.java:3883) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #37 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 23 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-360-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: 1874333546@qtp-158128739-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: IPC Server handler 1 on default port 43893 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #59 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 13 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 44 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 35557 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 43 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #44 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 27 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: DeletionService #3 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server idle connection scanner for port 39201 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 2 on default port 39201 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 4 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: hconnection-0x46f0727e-shared-pool-16 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: pool-376-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #60 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #46 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x46f0727e-shared-pool-17 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: BP-2060461976-172.31.12.81-1696250018691 heartbeating to localhost/127.0.0.1:35557 java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.IncrementalBlockReportManager.waitTillNextIBR(IncrementalBlockReportManager.java:158) org.apache.hadoop.hdfs.server.datanode.BPServiceActor.offerService(BPServiceActor.java:715) org.apache.hadoop.hdfs.server.datanode.BPServiceActor.run(BPServiceActor.java:851) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: org.apache.hadoop.hdfs.server.namenode.FSNamesystem$LazyPersistFileScrubber@4eada5de java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.server.namenode.FSNamesystem$LazyPersistFileScrubber.run(FSNamesystem.java:3975) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 30 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 40 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #58 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 3 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 37283 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-370-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 0 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 48 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 46 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 36 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 5 on default port 39201 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 11 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@546a9622 java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 0 on default port 42035 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 11 on default port 39201 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 5 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 3 on default port 43893 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-404-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: 152692919@qtp-128290155-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: IPC Server handler 40 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 3 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 49 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 31 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 24 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Client (556141606) connection to jenkins-hbase3.apache.org/172.31.12.81:40015 from jenkins java.lang.Object.wait(Native Method) org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1053) org.apache.hadoop.ipc.Client$Connection.run(Client.java:1097) Potentially hanging thread: IPC Server handler 22 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor@12e7432b java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor.run(HeartbeatManager.java:536) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-55 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-379697136_1 at /127.0.0.1:36458 [Waiting for operation #2] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 1 on default port 46763 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 31 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 21 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/cluster_8d5394c9-0df2-fcee-025a-99149dac8b29/dfs/data/data1) java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:627) Potentially hanging thread: 1872602723@qtp-1295801457-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: IPC Server handler 3 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@2c47e771 java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 48 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 8 on default port 44597 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 3 on default port 44597 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 12 on default port 39201 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 41 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 23 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: region-location-3 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-65 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 16 on default port 39201 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 31 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@66a9195 java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-39 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 0 on default port 39355 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 33 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: BP-2060461976-172.31.12.81-1696250018691 heartbeating to localhost/127.0.0.1:35557 java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.IncrementalBlockReportManager.waitTillNextIBR(IncrementalBlockReportManager.java:158) org.apache.hadoop.hdfs.server.datanode.BPServiceActor.offerService(BPServiceActor.java:715) org.apache.hadoop.hdfs.server.datanode.BPServiceActor.run(BPServiceActor.java:851) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 3 on default port 39201 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #63 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 18 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Thread[Thread-1301,5,FailOnTimeoutGroup] java.lang.Thread.sleep(Native Method) org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:694) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 1 on default port 39189 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 35557 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 46345 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_109360846_16 at /127.0.0.1:34826 [Waiting for operation #8] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ApplicationMasterLauncher #0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 3 on default port 46763 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 46 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-60 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: nioEventLoopGroup-10-1 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:803) io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457) io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 4 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 3 on default port 42035 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 45 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 17 on default port 39201 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 9 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 42035 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 6 on default port 39201 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: PacketResponder: BP-2107663488-172.31.12.81-1696249963873:blk_1073741871_1047, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 4 on default port 39201 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 35 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 21 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 8 on default port 39201 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 44 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: CacheReplicationMonitor(1274198381) sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) org.apache.hadoop.hdfs.server.blockmanagement.CacheReplicationMonitor.run(CacheReplicationMonitor.java:181) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:49524@0x5691222f sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:328) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$490/1348330431.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 10 on default port 39201 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 45 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@2fbb0065 java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/cluster_8d5394c9-0df2-fcee-025a-99149dac8b29/dfs/data/data3) java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:627) Potentially hanging thread: IPC Server handler 13 on default port 46763 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 47 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 1 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Thread-2052 java.io.FileInputStream.readBytes(Native Method) java.io.FileInputStream.read(FileInputStream.java:255) java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.io.BufferedInputStream.read(BufferedInputStream.java:345) sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:284) sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:326) sun.nio.cs.StreamDecoder.read(StreamDecoder.java:178) java.io.InputStreamReader.read(InputStreamReader.java:184) java.io.BufferedReader.fill(BufferedReader.java:161) java.io.BufferedReader.readLine(BufferedReader.java:324) java.io.BufferedReader.readLine(BufferedReader.java:389) org.apache.hadoop.util.Shell$1.run(Shell.java:955) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:49524@0x5691222f-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: IPC Server idle connection scanner for port 46763 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: nioEventLoopGroup-12-1 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:803) io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457) io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Thread-1343 java.lang.Thread.sleep(Native Method) org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:143) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-62 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 26 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 29 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 9 on default port 44597 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-372-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 39 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-61 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: pool-402-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 12 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #55 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 9 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 108209507@qtp-158128739-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:41143 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: IPC Server handler 21 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 39189 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 39 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 6 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 1 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #61 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: 322990203@qtp-263136829-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: IPC Server handler 34 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: region-location-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/cluster_8d5394c9-0df2-fcee-025a-99149dac8b29/dfs/data/data5/current/BP-2060461976-172.31.12.81-1696250018691 java.lang.Thread.sleep(Native Method) org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:205) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: 1995853231@qtp-1861216298-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: org.apache.hadoop.hdfs.server.datanode.DataXceiverServer@64c7e679 sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:421) sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:249) sun.nio.ch.ServerSocketAdaptor.accept(ServerSocketAdaptor.java:113) org.apache.hadoop.hdfs.net.TcpPeerServer.accept(TcpPeerServer.java:85) org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.run(DataXceiverServer.java:145) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 28 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #42 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: 2006615276@qtp-192977947-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: IPC Server handler 28 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 14 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: region-location-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 39 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 27 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 1 on default port 42035 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #54 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-48 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: Timer-59 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 45 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #66 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: 718221651@qtp-1375405873-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:46087 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: IPC Server handler 0 on default port 44597 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 7 on default port 39201 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 6 on default port 44597 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 12 on default port 46763 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 38421 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 8 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #47 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 6 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@520b5eb0[State = -1, empty queue] sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 17 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-356-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 10 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #67 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: org.apache.hadoop.hdfs.server.datanode.DataXceiverServer@531b9358 sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:421) sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:249) sun.nio.ch.ServerSocketAdaptor.accept(ServerSocketAdaptor.java:113) org.apache.hadoop.hdfs.net.TcpPeerServer.accept(TcpPeerServer.java:85) org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.run(DataXceiverServer.java:145) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-44 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor@42363617 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor.run(LeaseManager.java:533) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-49 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 16 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 14 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DeletionService #0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 23 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 28 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 1376067098@qtp-794069191-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:38473 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: IPC Server handler 11 on default port 46763 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 37283 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-379697136_1 at /127.0.0.1:57826 [Waiting for operation #2] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server idle connection scanner for port 39355 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 8 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 43 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 5 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 15 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 7 on default port 46763 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 16 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 15 on default port 46763 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 30 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 15 on default port 39201 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-63 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 43 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 36 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 46763 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 46345 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 748191775@qtp-1861216298-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:35403 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: IPC Server handler 1 on default port 46345 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: hconnection-0x46f0727e-shared-pool-14 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: nioEventLoopGroup-14-1 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:803) io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457) io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: pool-350-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ContainersLauncher #0 java.io.FileInputStream.readBytes(Native Method) java.io.FileInputStream.read(FileInputStream.java:255) java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.io.BufferedInputStream.read(BufferedInputStream.java:345) sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:284) sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:326) sun.nio.cs.StreamDecoder.read(StreamDecoder.java:178) java.io.InputStreamReader.read(InputStreamReader.java:184) java.io.BufferedReader.fill(BufferedReader.java:161) java.io.BufferedReader.read1(BufferedReader.java:212) java.io.BufferedReader.read(BufferedReader.java:286) org.apache.hadoop.util.Shell$ShellCommandExecutor.parseExecResult(Shell.java:1229) org.apache.hadoop.util.Shell.runCommand(Shell.java:984) org.apache.hadoop.util.Shell.run(Shell.java:884) org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1216) org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor.launchContainer(DefaultContainerExecutor.java:294) org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.launchContainer(ContainerLaunch.java:447) org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.call(ContainerLaunch.java:298) org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.call(ContainerLaunch.java:99) java.util.concurrent.FutureTask.run(FutureTask.java:266) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server idle connection scanner for port 45383 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: DeletionService #2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #40 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: pool-348-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/cluster_8d5394c9-0df2-fcee-025a-99149dac8b29/dfs/data/data6/current/BP-2060461976-172.31.12.81-1696250018691 java.lang.Thread.sleep(Native Method) org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:205) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server idle connection scanner for port 42035 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server idle connection scanner for port 46345 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 1 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 13 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 3 on default port 46345 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #43 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 37 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 11 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/cluster_8d5394c9-0df2-fcee-025a-99149dac8b29/dfs/data/data3/current/BP-2060461976-172.31.12.81-1696250018691 java.lang.Thread.sleep(Native Method) org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:205) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ContainersLauncher #1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 3 on default port 37283 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 19 on default port 46763 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.hdfs.server.blockmanagement.PendingReplicationBlocks$PendingReplicationMonitor@24f30da4 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.server.blockmanagement.PendingReplicationBlocks$PendingReplicationMonitor.run(PendingReplicationBlocks.java:244) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Thread[Thread-1336,5,FailOnTimeoutGroup] java.lang.Thread.sleep(Native Method) org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:694) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #49 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: LeaseRenewer:jenkins@localhost:35557 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:412) org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$600(LeaseRenewer.java:76) org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:308) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Client (556141606) connection to localhost/127.0.0.1:35557 from jenkins java.lang.Object.wait(Native Method) org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1053) org.apache.hadoop.ipc.Client$Connection.run(Client.java:1097) Potentially hanging thread: IPC Server handler 7 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 40 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-64 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 20 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-67 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 18 on default port 39201 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O server boss #68 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.jboss.netty.channel.socket.nio.NioServerBoss.select(NioServerBoss.java:163) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.NioServerBoss.run(NioServerBoss.java:42) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 24 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 49 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 38865 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: New I/O worker #45 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/cluster_8d5394c9-0df2-fcee-025a-99149dac8b29/dfs/data/data5) java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:627) Potentially hanging thread: IPC Server handler 22 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-47 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 34 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 35 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-54 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 42 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #65 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 37 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 1318214633@qtp-128290155-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:46739 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: IPC Server handler 4 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 20 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 16 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 26 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 10 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 36 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-366-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/cluster_8d5394c9-0df2-fcee-025a-99149dac8b29/dfs/data/data1/current/BP-2060461976-172.31.12.81-1696250018691 java.lang.Thread.sleep(Native Method) org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:205) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 2 on default port 35557 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeResourceMonitor@20e5ca43 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeResourceMonitor.run(FSNamesystem.java:3841) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: 891932820@qtp-1295801457-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:34007 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: IPC Server handler 9 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 4 on default port 37283 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 1 on default port 37283 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 10 on default port 46763 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 42 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-46 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 30 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 10 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 47 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 46 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 24 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #53 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Thread[Thread-1326,5,FailOnTimeoutGroup] java.lang.Thread.sleep(Native Method) org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:694) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: 1951373757@qtp-794069191-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: IPC Server handler 25 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 22 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 6 on default port 46763 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-50 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 47 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 15 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 44597 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server idle connection scanner for port 37283 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: New I/O worker #38 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: MutableQuantiles-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 38 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1259401672_16 at /127.0.0.1:49400 [Receiving block BP-2107663488-172.31.12.81-1696249963873:blk_1073741871_1047] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: BP-2060461976-172.31.12.81-1696250018691 heartbeating to localhost/127.0.0.1:35557 java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.IncrementalBlockReportManager.waitTillNextIBR(IncrementalBlockReportManager.java:158) org.apache.hadoop.hdfs.server.datanode.BPServiceActor.offerService(BPServiceActor.java:715) org.apache.hadoop.hdfs.server.datanode.BPServiceActor.run(BPServiceActor.java:851) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-52 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: New I/O worker #52 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: pool-353-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Client (556141606) connection to localhost/127.0.0.1:35557 from jenkins java.lang.Object.wait(Native Method) org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1053) org.apache.hadoop.ipc.Client$Connection.run(Client.java:1097) Potentially hanging thread: Container metrics unregistration java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: New I/O worker #50 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 2 on default port 43893 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 4 on default port 46763 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 19 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 25 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #36 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 1 on default port 44597 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #35 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@3bab4660 java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-66 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: Timer-43 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 14 on default port 46763 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 42 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@19e0fef6[State = -1, empty queue] sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 5 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-399-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #62 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server idle connection scanner for port 43827 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 29 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 8 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-40 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: PacketResponder: BP-2107663488-172.31.12.81-1696249963873:blk_1073741870_1046, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/cluster_8d5394c9-0df2-fcee-025a-99149dac8b29/dfs/data/data2/current/BP-2060461976-172.31.12.81-1696250018691 java.lang.Thread.sleep(Native Method) org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:205) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #39 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@54dcf481 java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #41 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 48 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/cluster_8d5394c9-0df2-fcee-025a-99149dac8b29/dfs/data/data2) java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:627) Potentially hanging thread: IPC Server handler 4 on default port 44597 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 13 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@217106ca[State = -1, empty queue] sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 14 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #48 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 25 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 19 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: HFileArchiver-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 44 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/cluster_8d5394c9-0df2-fcee-025a-99149dac8b29/dfs/data/data4/current/BP-2060461976-172.31.12.81-1696250018691 java.lang.Thread.sleep(Native Method) org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:205) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 16 on default port 46763 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-51 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 19 on default port 39201 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 33 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: DeletionService #1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/cluster_8d5394c9-0df2-fcee-025a-99149dac8b29/dfs/data/data6) java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:627) Potentially hanging thread: IPC Server handler 19 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 3 on default port 39189 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-57 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: ForkJoinPool-2-worker-7 sun.misc.Unsafe.park(Native Method) java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1824) java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1693) java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:49524@0x5691222f-SendThread(127.0.0.1:49524) sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:345) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1223) Potentially hanging thread: IPC Server handler 35 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1fc2141b-9f1f-112f-68e3-9b7275e4ccc8/cluster_8d5394c9-0df2-fcee-025a-99149dac8b29/dfs/data/data4) java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:627) Potentially hanging thread: IPC Server handler 32 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-362-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 6 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-58 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 26 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 15 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 43893 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 17 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 12 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-355-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 14 on default port 39201 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@282b007e java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS-EventLoopGroup-5-3 org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:209) org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:202) org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:306) org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:363) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@dbfcaa java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 4 on default port 42035 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 39189 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1259401672_16 at /127.0.0.1:49392 [Receiving block BP-2107663488-172.31.12.81-1696249963873:blk_1073741870_1046] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 11 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 8 on default port 46763 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 44597 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #64 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 17 on default port 46763 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.hdfs.server.datanode.DataXceiverServer@24383111 sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:421) sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:249) sun.nio.ch.ServerSocketAdaptor.accept(ServerSocketAdaptor.java:113) org.apache.hadoop.hdfs.net.TcpPeerServer.accept(TcpPeerServer.java:85) org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.run(DataXceiverServer.java:145) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 1 on default port 35557 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 38 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 649097587@qtp-1375405873-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: IPC Server handler 18 on default port 45383 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O server boss #51 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.jboss.netty.channel.socket.nio.NioServerBoss.select(NioServerBoss.java:163) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.NioServerBoss.run(NioServerBoss.java:42) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 1 on default port 39201 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: ForkJoinPool-2-worker-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1824) java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1693) java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) Potentially hanging thread: IPC Server handler 0 on default port 43827 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 34 on default port 38421 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 17 on default port 38865 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-42 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) - Thread LEAK? -, OpenFileDescriptor=1251 (was 903) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=517 (was 447) - SystemLoadAverage LEAK? -, ProcessCount=184 (was 181) - ProcessCount LEAK? -, AvailableMemoryMB=1184 (was 2242) 2023-10-02T12:34:08,772 WARN [Listener at jenkins-hbase3.apache.org/43893 {}] hbase.ResourceChecker(130): Thread=1305 is superior to 500 2023-10-02T12:34:08,772 WARN [Listener at jenkins-hbase3.apache.org/43893 {}] hbase.ResourceChecker(130): OpenFileDescriptor=1251 is superior to 1024 2023-10-02T12:34:08,780 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-10-02T12:34:08,781 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x042d2e96 to 127.0.0.1:49524 2023-10-02T12:34:08,781 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:08,787 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-10-02T12:34:08,787 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] util.JVMClusterUtil(257): Found active master hash=1889164881, stopped=false 2023-10-02T12:34:08,788 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] master.ServerManager(910): Cluster shutdown requested of master=jenkins-hbase3.apache.org,33927,1696249978185 2023-10-02T12:34:08,789 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] procedure2.ProcedureExecutor(630): Stopping 2023-10-02T12:34:08,789 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/2/running 2023-10-02T12:34:08,789 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35953-0x1017329d6f10005, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/2/running 2023-10-02T12:34:08,790 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-10-02T12:34:08,790 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35953-0x1017329d6f10005, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-10-02T12:34:08,790 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35953-0x1017329d6f10005, quorum=127.0.0.1:49524, baseZNode=/2 Set watcher on znode that does not yet exist, /2/running 2023-10-02T12:34:08,790 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Set watcher on znode that does not yet exist, /2/running 2023-10-02T12:34:08,790 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:08,791 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,35953,1696249978347' ***** 2023-10-02T12:34:08,791 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] regionserver.HRegionServer(2531): STOPPED: Shutdown requested 2023-10-02T12:34:08,791 INFO [RS:0;jenkins-hbase3:35953 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-10-02T12:34:08,815 INFO [RS:0;jenkins-hbase3:35953 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@74598388{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-10-02T12:34:08,821 INFO [RS:0;jenkins-hbase3:35953 {}] server.AbstractConnector(383): Stopped ServerConnector@3ca329d6{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-10-02T12:34:08,821 INFO [RS:0;jenkins-hbase3:35953 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-10-02T12:34:08,827 INFO [RS:0;jenkins-hbase3:35953 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@3291cb0b{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-10-02T12:34:08,829 INFO [RS:0;jenkins-hbase3:35953 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@1c32c707{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/hadoop.log.dir/,STOPPED} 2023-10-02T12:34:08,832 INFO [RS:0;jenkins-hbase3:35953 {}] regionserver.HeapMemoryManager(220): Stopping 2023-10-02T12:34:08,832 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-10-02T12:34:08,833 INFO [RS:0;jenkins-hbase3:35953 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2023-10-02T12:34:08,833 INFO [RS:0;jenkins-hbase3:35953 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-10-02T12:34:08,833 INFO [RS:0;jenkins-hbase3:35953 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-10-02T12:34:08,833 INFO [RS:0;jenkins-hbase3:35953 {}] regionserver.HRegionServer(3528): Received CLOSE for c28d048d5108c73bc091dea0057bf483 2023-10-02T12:34:08,835 INFO [RS:0;jenkins-hbase3:35953 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,35953,1696249978347 2023-10-02T12:34:08,835 DEBUG [RS:0;jenkins-hbase3:35953 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:08,840 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing c28d048d5108c73bc091dea0057bf483, disabling compactions & flushes 2023-10-02T12:34:08,840 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1696249981588.c28d048d5108c73bc091dea0057bf483. 2023-10-02T12:34:08,840 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1696249981588.c28d048d5108c73bc091dea0057bf483. 2023-10-02T12:34:08,840 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1696249981588.c28d048d5108c73bc091dea0057bf483. after waiting 0 ms 2023-10-02T12:34:08,840 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1696249981588.c28d048d5108c73bc091dea0057bf483. 2023-10-02T12:34:08,840 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing c28d048d5108c73bc091dea0057bf483 1/1 column families, dataSize=78 B heapSize=488 B 2023-10-02T12:34:08,841 INFO [RS:0;jenkins-hbase3:35953 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-10-02T12:34:08,841 INFO [RS:0;jenkins-hbase3:35953 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-10-02T12:34:08,841 INFO [RS:0;jenkins-hbase3:35953 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-10-02T12:34:08,841 INFO [RS:0;jenkins-hbase3:35953 {}] regionserver.HRegionServer(3528): Received CLOSE for 1588230740 2023-10-02T12:34:08,842 INFO [RS:0;jenkins-hbase3:35953 {}] regionserver.HRegionServer(1579): Waiting on 2 regions to close 2023-10-02T12:34:08,842 DEBUG [RS:0;jenkins-hbase3:35953 {}] regionserver.HRegionServer(1583): Online Regions={1588230740=hbase:meta,,1.1588230740, c28d048d5108c73bc091dea0057bf483=hbase:namespace,,1696249981588.c28d048d5108c73bc091dea0057bf483.} 2023-10-02T12:34:08,852 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-10-02T12:34:08,852 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-10-02T12:34:08,852 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-10-02T12:34:08,852 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-10-02T12:34:08,852 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-10-02T12:34:08,853 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=1.26 KB heapSize=2.89 KB 2023-10-02T12:34:08,857 DEBUG [RS:0;jenkins-hbase3:35953 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, c28d048d5108c73bc091dea0057bf483 2023-10-02T12:34:08,857 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-10-02T12:34:08,888 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/data/hbase/namespace/c28d048d5108c73bc091dea0057bf483/.tmp/info/bd12b5b1296d475bb4666c636cd4d1bc is 45, key is default/info:d/1696249982910/Put/seqid=0 2023-10-02T12:34:08,902 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/data/hbase/meta/1588230740/.tmp/info/e4adc0fa45964f639b4794c8a4b012ec is 143, key is hbase:namespace,,1696249981588.c28d048d5108c73bc091dea0057bf483./info:regioninfo/1696249982809/Put/seqid=0 2023-10-02T12:34:09,057 DEBUG [RS:0;jenkins-hbase3:35953 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, c28d048d5108c73bc091dea0057bf483 2023-10-02T12:34:09,257 DEBUG [RS:0;jenkins-hbase3:35953 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, c28d048d5108c73bc091dea0057bf483 2023-10-02T12:34:09,304 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/data/hbase/namespace/c28d048d5108c73bc091dea0057bf483/.tmp/info/bd12b5b1296d475bb4666c636cd4d1bc 2023-10-02T12:34:09,312 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.17 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/data/hbase/meta/1588230740/.tmp/info/e4adc0fa45964f639b4794c8a4b012ec 2023-10-02T12:34:09,339 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/data/hbase/namespace/c28d048d5108c73bc091dea0057bf483/.tmp/info/bd12b5b1296d475bb4666c636cd4d1bc as hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/data/hbase/namespace/c28d048d5108c73bc091dea0057bf483/info/bd12b5b1296d475bb4666c636cd4d1bc 2023-10-02T12:34:09,347 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/data/hbase/namespace/c28d048d5108c73bc091dea0057bf483/info/bd12b5b1296d475bb4666c636cd4d1bc, entries=2, sequenceid=6, filesize=4.9 K 2023-10-02T12:34:09,348 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for c28d048d5108c73bc091dea0057bf483 in 508ms, sequenceid=6, compaction requested=false 2023-10-02T12:34:09,358 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/data/hbase/meta/1588230740/.tmp/table/0e942ab2edaa4efda998166039d23900 is 51, key is hbase:namespace/table:state/1696249982822/Put/seqid=0 2023-10-02T12:34:09,370 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46097/tmp/wal/data/hbase/namespace/c28d048d5108c73bc091dea0057bf483/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2023-10-02T12:34:09,371 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=94 B at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/data/hbase/meta/1588230740/.tmp/table/0e942ab2edaa4efda998166039d23900 2023-10-02T12:34:09,371 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:34:09,374 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1696249981588.c28d048d5108c73bc091dea0057bf483. 2023-10-02T12:34:09,375 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for c28d048d5108c73bc091dea0057bf483: 2023-10-02T12:34:09,375 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1696249981588.c28d048d5108c73bc091dea0057bf483. 2023-10-02T12:34:09,397 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/data/hbase/meta/1588230740/.tmp/info/e4adc0fa45964f639b4794c8a4b012ec as hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/data/hbase/meta/1588230740/info/e4adc0fa45964f639b4794c8a4b012ec 2023-10-02T12:34:09,405 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/data/hbase/meta/1588230740/info/e4adc0fa45964f639b4794c8a4b012ec, entries=10, sequenceid=9, filesize=6.4 K 2023-10-02T12:34:09,406 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/data/hbase/meta/1588230740/.tmp/table/0e942ab2edaa4efda998166039d23900 as hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/data/hbase/meta/1588230740/table/0e942ab2edaa4efda998166039d23900 2023-10-02T12:34:09,407 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-10-02T12:34:09,407 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-10-02T12:34:09,414 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/data/hbase/meta/1588230740/table/0e942ab2edaa4efda998166039d23900, entries=2, sequenceid=9, filesize=5.1 K 2023-10-02T12:34:09,415 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~1.26 KB/1290, heapSize ~2.61 KB/2672, currentSize=0 B/0 for 1588230740 in 562ms, sequenceid=9, compaction requested=false 2023-10-02T12:34:09,430 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46097/tmp/wal/data/hbase/meta/1588230740/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2023-10-02T12:34:09,431 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:34:09,431 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-10-02T12:34:09,431 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-10-02T12:34:09,431 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-10-02T12:34:09,431 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-10-02T12:34:09,458 INFO [RS:0;jenkins-hbase3:35953 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,35953,1696249978347; all regions closed. 2023-10-02T12:34:09,471 DEBUG [RS:0;jenkins-hbase3:35953 {}] wal.AbstractFSWAL(1062): Moved 1 WAL file(s) to /tmp/wal/oldWALs 2023-10-02T12:34:09,471 INFO [RS:0;jenkins-hbase3:35953 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C35953%2C1696249978347.meta:.meta(num 1696249981507) 2023-10-02T12:34:09,480 DEBUG [RS:0;jenkins-hbase3:35953 {}] wal.AbstractFSWAL(1062): Moved 1 WAL file(s) to /tmp/wal/oldWALs 2023-10-02T12:34:09,480 INFO [RS:0;jenkins-hbase3:35953 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C35953%2C1696249978347:(num 1696249980544) 2023-10-02T12:34:09,480 DEBUG [RS:0;jenkins-hbase3:35953 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:09,480 INFO [RS:0;jenkins-hbase3:35953 {}] regionserver.LeaseManager(133): Closed leases 2023-10-02T12:34:09,481 INFO [RS:0;jenkins-hbase3:35953 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2023-10-02T12:34:09,481 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-10-02T12:34:09,483 INFO [RS:0;jenkins-hbase3:35953 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:35953 2023-10-02T12:34:09,492 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35953-0x1017329d6f10005, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/2/rs/jenkins-hbase3.apache.org,35953,1696249978347 2023-10-02T12:34:09,492 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/rs 2023-10-02T12:34:09,496 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,35953,1696249978347] 2023-10-02T12:34:09,496 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,35953,1696249978347; numProcessing=1 2023-10-02T12:34:09,498 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(224): Node /2/draining/jenkins-hbase3.apache.org,35953,1696249978347 already deleted, retry=false 2023-10-02T12:34:09,498 INFO [RegionServerTracker-0 {}] master.ServerManager(569): Cluster shutdown set; jenkins-hbase3.apache.org,35953,1696249978347 expired; onlineServers=0 2023-10-02T12:34:09,498 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,33927,1696249978185' ***** 2023-10-02T12:34:09,498 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2531): STOPPED: Cluster shutdown set; onlineServer=0 2023-10-02T12:34:09,500 DEBUG [M:0;jenkins-hbase3:33927 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e5021c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-10-02T12:34:09,500 INFO [M:0;jenkins-hbase3:33927 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-10-02T12:34:09,520 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/2/master 2023-10-02T12:34:09,520 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-10-02T12:34:09,522 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Set watcher on znode that does not yet exist, /2/master 2023-10-02T12:34:09,638 INFO [RS:0;jenkins-hbase3:35953 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,35953,1696249978347; zookeeper connection closed. 2023-10-02T12:34:09,642 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35953-0x1017329d6f10005, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-10-02T12:34:09,642 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35953-0x1017329d6f10005, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-10-02T12:34:09,642 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1676fc32 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1676fc32 2023-10-02T12:34:09,646 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-10-02T12:34:09,692 INFO [M:0;jenkins-hbase3:33927 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@63ff4594{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-10-02T12:34:09,693 INFO [M:0;jenkins-hbase3:33927 {}] server.AbstractConnector(383): Stopped ServerConnector@37a50ea5{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-10-02T12:34:09,693 INFO [M:0;jenkins-hbase3:33927 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-10-02T12:34:09,694 INFO [M:0;jenkins-hbase3:33927 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@13d2f11e{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-10-02T12:34:09,712 INFO [M:0;jenkins-hbase3:33927 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@46194509{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/hadoop.log.dir/,STOPPED} 2023-10-02T12:34:09,713 INFO [M:0;jenkins-hbase3:33927 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,33927,1696249978185 2023-10-02T12:34:09,713 INFO [M:0;jenkins-hbase3:33927 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,33927,1696249978185; all regions closed. 2023-10-02T12:34:09,713 DEBUG [M:0;jenkins-hbase3:33927 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:09,713 INFO [M:0;jenkins-hbase3:33927 {}] master.HMaster(1665): Stopping master jetty server 2023-10-02T12:34:09,714 INFO [M:0;jenkins-hbase3:33927 {}] server.AbstractConnector(383): Stopped ServerConnector@3a1f71b3{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-10-02T12:34:09,715 DEBUG [M:0;jenkins-hbase3:33927 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-10-02T12:34:09,715 INFO [M:0;jenkins-hbase3:33927 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-10-02T12:34:09,715 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-10-02T12:34:09,715 DEBUG [M:0;jenkins-hbase3:33927 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x747ec952 to 127.0.0.1:49524 2023-10-02T12:34:09,715 DEBUG [M:0;jenkins-hbase3:33927 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:09,716 DEBUG [M:0;jenkins-hbase3:33927 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-10-02T12:34:09,716 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1696249980314 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1696249980314,5,FailOnTimeoutGroup] 2023-10-02T12:34:09,716 INFO [M:0;jenkins-hbase3:33927 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-10-02T12:34:09,716 DEBUG [M:0;jenkins-hbase3:33927 {}] master.HMaster(1688): Stopping service threads 2023-10-02T12:34:09,716 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1696249980310 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1696249980310,5,FailOnTimeoutGroup] 2023-10-02T12:34:09,716 INFO [M:0;jenkins-hbase3:33927 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-10-02T12:34:09,716 ERROR [M:0;jenkins-hbase3:33927 {}] procedure2.ProcedureExecutor(654): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] 2023-10-02T12:34:09,717 INFO [M:0;jenkins-hbase3:33927 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-10-02T12:34:09,717 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-10-02T12:34:09,718 DEBUG [M:0;jenkins-hbase3:33927 {}] zookeeper.ZKUtil(347): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Unable to get data of znode /2/master because node does not exist (not an error) 2023-10-02T12:34:09,718 WARN [M:0;jenkins-hbase3:33927 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-10-02T12:34:09,718 INFO [M:0;jenkins-hbase3:33927 {}] assignment.AssignmentManager(382): Stopping assignment manager 2023-10-02T12:34:09,718 INFO [M:0;jenkins-hbase3:33927 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-10-02T12:34:09,721 DEBUG [M:0;jenkins-hbase3:33927 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-10-02T12:34:09,733 INFO [M:0;jenkins-hbase3:33927 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:34:09,733 DEBUG [M:0;jenkins-hbase3:33927 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:34:09,733 DEBUG [M:0;jenkins-hbase3:33927 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-10-02T12:34:09,733 DEBUG [M:0;jenkins-hbase3:33927 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:34:09,734 INFO [M:0;jenkins-hbase3:33927 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=25.76 KB heapSize=33.30 KB 2023-10-02T12:34:09,747 DEBUG [M:0;jenkins-hbase3:33927 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e87ef4e15faa44f88db3a619d84c6f4b is 88, key is hbase:meta,,1/info:sn/1696249981333/Put/seqid=0 2023-10-02T12:34:10,153 INFO [M:0;jenkins-hbase3:33927 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e87ef4e15faa44f88db3a619d84c6f4b 2023-10-02T12:34:10,191 DEBUG [M:0;jenkins-hbase3:33927 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/086fb0a5ce81468291d48d4f9107042e is 241, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1696249981570/Put/seqid=0 2023-10-02T12:34:10,354 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2023-10-02T12:34:10,598 INFO [M:0;jenkins-hbase3:33927 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25.17 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/086fb0a5ce81468291d48d4f9107042e 2023-10-02T12:34:10,603 INFO [M:0;jenkins-hbase3:33927 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 086fb0a5ce81468291d48d4f9107042e 2023-10-02T12:34:10,617 DEBUG [M:0;jenkins-hbase3:33927 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/69469dea0ceb4580b8b9da079aa2f2d2 is 82, key is jenkins-hbase3.apache.org,35953,1696249978347/rs:state/1696249980370/Put/seqid=0 2023-10-02T12:34:11,022 INFO [M:0;jenkins-hbase3:33927 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/69469dea0ceb4580b8b9da079aa2f2d2 2023-10-02T12:34:11,029 DEBUG [M:0;jenkins-hbase3:33927 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e87ef4e15faa44f88db3a619d84c6f4b as hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e87ef4e15faa44f88db3a619d84c6f4b 2023-10-02T12:34:11,039 INFO [M:0;jenkins-hbase3:33927 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e87ef4e15faa44f88db3a619d84c6f4b, entries=8, sequenceid=76, filesize=5.5 K 2023-10-02T12:34:11,040 DEBUG [M:0;jenkins-hbase3:33927 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/086fb0a5ce81468291d48d4f9107042e as hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/086fb0a5ce81468291d48d4f9107042e 2023-10-02T12:34:11,046 INFO [M:0;jenkins-hbase3:33927 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 086fb0a5ce81468291d48d4f9107042e 2023-10-02T12:34:11,046 INFO [M:0;jenkins-hbase3:33927 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/086fb0a5ce81468291d48d4f9107042e, entries=8, sequenceid=76, filesize=5.4 K 2023-10-02T12:34:11,047 DEBUG [M:0;jenkins-hbase3:33927 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/69469dea0ceb4580b8b9da079aa2f2d2 as hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/69469dea0ceb4580b8b9da079aa2f2d2 2023-10-02T12:34:11,052 INFO [M:0;jenkins-hbase3:33927 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:46097/user/jenkins/test-data/e9eee57f-263f-d1ba-63f3-2af27bcd4648/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/69469dea0ceb4580b8b9da079aa2f2d2, entries=1, sequenceid=76, filesize=5.1 K 2023-10-02T12:34:11,053 INFO [M:0;jenkins-hbase3:33927 {}] regionserver.HRegion(3022): Finished flush of dataSize ~25.76 KB/26381, heapSize ~33.01 KB/33800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1320ms, sequenceid=76, compaction requested=false 2023-10-02T12:34:11,062 INFO [M:0;jenkins-hbase3:33927 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:34:11,063 DEBUG [M:0;jenkins-hbase3:33927 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-10-02T12:34:11,067 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /tmp/wal/MasterData/WALs/jenkins-hbase3.apache.org,33927,1696249978185/jenkins-hbase3.apache.org%2C33927%2C1696249978185.1696249980131 not finished, retry = 0 2023-10-02T12:34:11,168 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-10-02T12:34:11,169 INFO [M:0;jenkins-hbase3:33927 {}] flush.MasterFlushTableProcedureManager(85): stop: server shutting down. 2023-10-02T12:34:11,169 INFO [M:0;jenkins-hbase3:33927 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-10-02T12:34:11,172 INFO [M:0;jenkins-hbase3:33927 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:33927 2023-10-02T12:34:11,175 DEBUG [M:0;jenkins-hbase3:33927 {}] zookeeper.RecoverableZooKeeper(224): Node /2/rs/jenkins-hbase3.apache.org,33927,1696249978185 already deleted, retry=false 2023-10-02T12:34:11,279 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-10-02T12:34:11,279 INFO [M:0;jenkins-hbase3:33927 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,33927,1696249978185; zookeeper connection closed. 2023-10-02T12:34:11,279 DEBUG [Listener at localhost/37315-EventThread {}] zookeeper.ZKWatcher(604): master:33927-0x1017329d6f10004, quorum=127.0.0.1:49524, baseZNode=/2 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-10-02T12:34:11,283 WARN [Listener at jenkins-hbase3.apache.org/43893 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-10-02T12:34:11,316 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-10-02T12:34:11,428 WARN [BP-1900449672-172.31.12.81-1696249977243 heartbeating to localhost/127.0.0.1:46097 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-10-02T12:34:11,428 WARN [BP-1900449672-172.31.12.81-1696249977243 heartbeating to localhost/127.0.0.1:46097 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-1900449672-172.31.12.81-1696249977243 (Datanode Uuid 09f18d7a-3acb-4d41-bd6e-50104e61a66d) service to localhost/127.0.0.1:46097 2023-10-02T12:34:11,431 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/cluster_e48cb9c6-a4ad-e8f6-28ba-f199054ea92b/dfs/data/data6/current/BP-1900449672-172.31.12.81-1696249977243 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-10-02T12:34:11,431 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/cluster_e48cb9c6-a4ad-e8f6-28ba-f199054ea92b/dfs/data/data5/current/BP-1900449672-172.31.12.81-1696249977243 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-10-02T12:34:11,439 WARN [Listener at jenkins-hbase3.apache.org/43893 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-10-02T12:34:11,461 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-10-02T12:34:11,571 WARN [BP-1900449672-172.31.12.81-1696249977243 heartbeating to localhost/127.0.0.1:46097 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-10-02T12:34:11,572 WARN [BP-1900449672-172.31.12.81-1696249977243 heartbeating to localhost/127.0.0.1:46097 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-1900449672-172.31.12.81-1696249977243 (Datanode Uuid c7b5b48b-45b9-4519-bab6-6809c95a1272) service to localhost/127.0.0.1:46097 2023-10-02T12:34:11,572 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/cluster_e48cb9c6-a4ad-e8f6-28ba-f199054ea92b/dfs/data/data3/current/BP-1900449672-172.31.12.81-1696249977243 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-10-02T12:34:11,573 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/cluster_e48cb9c6-a4ad-e8f6-28ba-f199054ea92b/dfs/data/data4/current/BP-1900449672-172.31.12.81-1696249977243 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-10-02T12:34:11,578 WARN [Listener at jenkins-hbase3.apache.org/43893 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-10-02T12:34:11,600 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-10-02T12:34:11,712 WARN [BP-1900449672-172.31.12.81-1696249977243 heartbeating to localhost/127.0.0.1:46097 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-10-02T12:34:11,712 WARN [BP-1900449672-172.31.12.81-1696249977243 heartbeating to localhost/127.0.0.1:46097 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-1900449672-172.31.12.81-1696249977243 (Datanode Uuid a0d34491-d14b-4a21-99d8-7739ed2881d7) service to localhost/127.0.0.1:46097 2023-10-02T12:34:11,713 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/cluster_e48cb9c6-a4ad-e8f6-28ba-f199054ea92b/dfs/data/data2/current/BP-1900449672-172.31.12.81-1696249977243 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-10-02T12:34:11,714 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/33f11059-3545-a1dc-5412-7d12d9a27569/cluster_e48cb9c6-a4ad-e8f6-28ba-f199054ea92b/dfs/data/data1/current/BP-1900449672-172.31.12.81-1696249977243 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-10-02T12:34:11,774 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-10-02T12:34:11,920 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-10-02T12:34:11,920 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-10-02T12:34:11,920 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-02T12:34:11,920 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2e685c49 to 127.0.0.1:49524 2023-10-02T12:34:11,920 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:11,920 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-10-02T12:34:11,920 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] util.JVMClusterUtil(257): Found active master hash=174330860, stopped=false 2023-10-02T12:34:11,920 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] master.ServerManager(910): Cluster shutdown requested of master=jenkins-hbase3.apache.org,45341,1696249969200 2023-10-02T12:34:11,922 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2023-10-02T12:34:11,922 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2023-10-02T12:34:11,922 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:34:11,922 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] procedure2.ProcedureExecutor(630): Stopping 2023-10-02T12:34:11,922 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:34:11,922 DEBUG [Listener at jenkins-hbase3.apache.org/43893 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:11,922 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,38375,1696249972520' ***** 2023-10-02T12:34:11,922 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] regionserver.HRegionServer(2531): STOPPED: Shutdown requested 2023-10-02T12:34:11,922 INFO [RS:0;jenkins-hbase3:38375 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-10-02T12:34:11,923 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-10-02T12:34:11,929 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-10-02T12:34:11,929 INFO [RS:0;jenkins-hbase3:38375 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@24d48c2{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-10-02T12:34:11,929 INFO [RS:0;jenkins-hbase3:38375 {}] server.AbstractConnector(383): Stopped ServerConnector@3a93225f{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-10-02T12:34:11,929 INFO [RS:0;jenkins-hbase3:38375 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-10-02T12:34:11,931 INFO [RS:0;jenkins-hbase3:38375 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@3bce61c2{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-10-02T12:34:11,932 INFO [RS:0;jenkins-hbase3:38375 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@322f357a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/hadoop.log.dir/,STOPPED} 2023-10-02T12:34:11,933 INFO [RS:0;jenkins-hbase3:38375 {}] regionserver.HeapMemoryManager(220): Stopping 2023-10-02T12:34:11,933 INFO [RS:0;jenkins-hbase3:38375 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2023-10-02T12:34:11,933 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-10-02T12:34:11,933 INFO [RS:0;jenkins-hbase3:38375 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-10-02T12:34:11,934 INFO [RS:0;jenkins-hbase3:38375 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-10-02T12:34:11,934 INFO [RS:0;jenkins-hbase3:38375 {}] regionserver.HRegionServer(3528): Received CLOSE for 7fa9a98238e4b256d46f306c833fe67e 2023-10-02T12:34:11,934 INFO [RS:0;jenkins-hbase3:38375 {}] regionserver.HRegionServer(3528): Received CLOSE for 73dd4d167cdf4d422164d1f17ee13a19 2023-10-02T12:34:11,934 INFO [RS:0;jenkins-hbase3:38375 {}] regionserver.HRegionServer(3528): Received CLOSE for 737b8e61c96d872ea2ba4221e6c477e8 2023-10-02T12:34:11,934 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 7fa9a98238e4b256d46f306c833fe67e, disabling compactions & flushes 2023-10-02T12:34:11,934 INFO [RS:0;jenkins-hbase3:38375 {}] regionserver.HRegionServer(3528): Received CLOSE for df86f7708dbdd71bc4105e1e70256952 2023-10-02T12:34:11,934 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system,,1696249994709.7fa9a98238e4b256d46f306c833fe67e. 2023-10-02T12:34:11,934 INFO [RS:0;jenkins-hbase3:38375 {}] regionserver.HRegionServer(3528): Received CLOSE for 1d3e651667b9d2576aabb8501f40423e 2023-10-02T12:34:11,934 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1696249994709.7fa9a98238e4b256d46f306c833fe67e. 2023-10-02T12:34:11,934 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1696249994709.7fa9a98238e4b256d46f306c833fe67e. after waiting 0 ms 2023-10-02T12:34:11,934 INFO [RS:0;jenkins-hbase3:38375 {}] regionserver.HRegionServer(3528): Received CLOSE for 567521894cd008d1b0178b9ce1b98f67 2023-10-02T12:34:11,934 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1696249994709.7fa9a98238e4b256d46f306c833fe67e. 2023-10-02T12:34:11,934 INFO [RS:0;jenkins-hbase3:38375 {}] regionserver.HRegionServer(3528): Received CLOSE for ced4de67d56921f0b2e4ff571d6ad352 2023-10-02T12:34:11,934 INFO [RS:0;jenkins-hbase3:38375 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:34:11,934 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 7fa9a98238e4b256d46f306c833fe67e 2/2 column families, dataSize=985 B heapSize=1.97 KB 2023-10-02T12:34:11,934 INFO [RS:0;jenkins-hbase3:38375 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-02T12:34:11,934 DEBUG [RS:0;jenkins-hbase3:38375 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:11,934 INFO [RS:0;jenkins-hbase3:38375 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-10-02T12:34:11,934 INFO [RS:0;jenkins-hbase3:38375 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-10-02T12:34:11,935 INFO [RS:0;jenkins-hbase3:38375 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-10-02T12:34:11,935 INFO [RS:0;jenkins-hbase3:38375 {}] regionserver.HRegionServer(3528): Received CLOSE for 1588230740 2023-10-02T12:34:11,943 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-10-02T12:34:11,946 INFO [RS:0;jenkins-hbase3:38375 {}] regionserver.HRegionServer(1579): Waiting on 8 regions to close 2023-10-02T12:34:11,946 DEBUG [RS:0;jenkins-hbase3:38375 {}] regionserver.HRegionServer(1583): Online Regions={7fa9a98238e4b256d46f306c833fe67e=backup:system,,1696249994709.7fa9a98238e4b256d46f306c833fe67e., 73dd4d167cdf4d422164d1f17ee13a19=test-1696249987851,,1696249988357.73dd4d167cdf4d422164d1f17ee13a19., 737b8e61c96d872ea2ba4221e6c477e8=backup:system_bulk,,1696249996848.737b8e61c96d872ea2ba4221e6c477e8., df86f7708dbdd71bc4105e1e70256952=hbase:namespace,,1696249976062.df86f7708dbdd71bc4105e1e70256952., 1588230740=hbase:meta,,1.1588230740, 1d3e651667b9d2576aabb8501f40423e=ns3:test-16962499878512,,1696249991854.1d3e651667b9d2576aabb8501f40423e., 567521894cd008d1b0178b9ce1b98f67=ns2:test-16962499878511,,1696249990628.567521894cd008d1b0178b9ce1b98f67., ced4de67d56921f0b2e4ff571d6ad352=ns4:test-16962499878513,,1696249992983.ced4de67d56921f0b2e4ff571d6ad352.} 2023-10-02T12:34:11,951 DEBUG [RS:0;jenkins-hbase3:38375 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 1d3e651667b9d2576aabb8501f40423e, 567521894cd008d1b0178b9ce1b98f67, 737b8e61c96d872ea2ba4221e6c477e8, 73dd4d167cdf4d422164d1f17ee13a19, 7fa9a98238e4b256d46f306c833fe67e, ced4de67d56921f0b2e4ff571d6ad352, df86f7708dbdd71bc4105e1e70256952 2023-10-02T12:34:11,951 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-10-02T12:34:11,951 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-10-02T12:34:11,951 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-10-02T12:34:11,951 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-10-02T12:34:11,951 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-10-02T12:34:11,951 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=13.51 KB heapSize=24 KB 2023-10-02T12:34:11,958 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/.tmp/meta/f663c55d2dac469094c765fed1b2662b is 169, key is trslm:hdfs://localhost:46097/backupUT\x00test-1696249987851/meta:log-roll-map/1696250018221/Put/seqid=0 2023-10-02T12:34:11,968 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=27 (bloomFilter=true), to=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/.tmp/meta/f663c55d2dac469094c765fed1b2662b 2023-10-02T12:34:11,980 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/hbase/meta/1588230740/.tmp/info/53cdc018356246a79278d9625c3af3ea is 159, key is ns2:test-16962499878511,,1696249990628.567521894cd008d1b0178b9ce1b98f67./info:regioninfo/1696249991452/Put/seqid=0 2023-10-02T12:34:11,983 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/.tmp/session/e8edc45855cd424eac04a5c87850fc7c is 310, key is session:backup_1696250010959/session:context/1696250018651/Put/seqid=0 2023-10-02T12:34:12,076 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-10-02T12:34:12,076 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-10-02T12:34:12,151 DEBUG [RS:0;jenkins-hbase3:38375 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 1d3e651667b9d2576aabb8501f40423e, 567521894cd008d1b0178b9ce1b98f67, 737b8e61c96d872ea2ba4221e6c477e8, 73dd4d167cdf4d422164d1f17ee13a19, 7fa9a98238e4b256d46f306c833fe67e, ced4de67d56921f0b2e4ff571d6ad352, df86f7708dbdd71bc4105e1e70256952 2023-10-02T12:34:12,351 DEBUG [RS:0;jenkins-hbase3:38375 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 1d3e651667b9d2576aabb8501f40423e, 567521894cd008d1b0178b9ce1b98f67, 737b8e61c96d872ea2ba4221e6c477e8, 73dd4d167cdf4d422164d1f17ee13a19, 7fa9a98238e4b256d46f306c833fe67e, ced4de67d56921f0b2e4ff571d6ad352, df86f7708dbdd71bc4105e1e70256952 2023-10-02T12:34:12,391 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.12 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/hbase/meta/1588230740/.tmp/info/53cdc018356246a79278d9625c3af3ea 2023-10-02T12:34:12,391 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=534 B at sequenceid=27 (bloomFilter=true), to=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/.tmp/session/e8edc45855cd424eac04a5c87850fc7c 2023-10-02T12:34:12,398 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/.tmp/meta/f663c55d2dac469094c765fed1b2662b as hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/meta/f663c55d2dac469094c765fed1b2662b 2023-10-02T12:34:12,405 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/meta/f663c55d2dac469094c765fed1b2662b, entries=4, sequenceid=27, filesize=5.5 K 2023-10-02T12:34:12,406 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/.tmp/session/e8edc45855cd424eac04a5c87850fc7c as hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/session/e8edc45855cd424eac04a5c87850fc7c 2023-10-02T12:34:12,411 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/hbase/meta/1588230740/.tmp/rep_barrier/e6be557979eb479b87cc13a6a0802af4 is 90, key is table1,,1696250006368.0daec27bd9d108cbf260f6b50d5177c8./rep_barrier:/1696250010817/DeleteFamily/seqid=0 2023-10-02T12:34:12,412 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/session/e8edc45855cd424eac04a5c87850fc7c, entries=2, sequenceid=27, filesize=5.3 K 2023-10-02T12:34:12,413 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~985 B/985, heapSize ~1.94 KB/1984, currentSize=0 B/0 for 7fa9a98238e4b256d46f306c833fe67e in 479ms, sequenceid=27, compaction requested=true 2023-10-02T12:34:12,444 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system/7fa9a98238e4b256d46f306c833fe67e/recovered.edits/30.seqid, newMaxSeqId=30, maxSeqId=1 2023-10-02T12:34:12,446 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:34:12,448 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system,,1696249994709.7fa9a98238e4b256d46f306c833fe67e. 2023-10-02T12:34:12,448 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 7fa9a98238e4b256d46f306c833fe67e: 2023-10-02T12:34:12,448 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1696249994709.7fa9a98238e4b256d46f306c833fe67e. 2023-10-02T12:34:12,448 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 73dd4d167cdf4d422164d1f17ee13a19, disabling compactions & flushes 2023-10-02T12:34:12,448 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region test-1696249987851,,1696249988357.73dd4d167cdf4d422164d1f17ee13a19. 2023-10-02T12:34:12,448 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1696249987851,,1696249988357.73dd4d167cdf4d422164d1f17ee13a19. 2023-10-02T12:34:12,448 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on test-1696249987851,,1696249988357.73dd4d167cdf4d422164d1f17ee13a19. after waiting 0 ms 2023-10-02T12:34:12,448 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region test-1696249987851,,1696249988357.73dd4d167cdf4d422164d1f17ee13a19. 2023-10-02T12:34:12,455 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/default/test-1696249987851/73dd4d167cdf4d422164d1f17ee13a19/recovered.edits/107.seqid, newMaxSeqId=107, maxSeqId=1 2023-10-02T12:34:12,456 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:34:12,457 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed test-1696249987851,,1696249988357.73dd4d167cdf4d422164d1f17ee13a19. 2023-10-02T12:34:12,457 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 73dd4d167cdf4d422164d1f17ee13a19: 2023-10-02T12:34:12,457 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1696249987851,,1696249988357.73dd4d167cdf4d422164d1f17ee13a19. 2023-10-02T12:34:12,458 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 737b8e61c96d872ea2ba4221e6c477e8, disabling compactions & flushes 2023-10-02T12:34:12,458 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1696249996848.737b8e61c96d872ea2ba4221e6c477e8. 2023-10-02T12:34:12,458 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1696249996848.737b8e61c96d872ea2ba4221e6c477e8. 2023-10-02T12:34:12,458 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1696249996848.737b8e61c96d872ea2ba4221e6c477e8. after waiting 0 ms 2023-10-02T12:34:12,458 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1696249996848.737b8e61c96d872ea2ba4221e6c477e8. 2023-10-02T12:34:12,462 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/backup/system_bulk/737b8e61c96d872ea2ba4221e6c477e8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-10-02T12:34:12,463 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:34:12,464 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system_bulk,,1696249996848.737b8e61c96d872ea2ba4221e6c477e8. 2023-10-02T12:34:12,464 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 737b8e61c96d872ea2ba4221e6c477e8: 2023-10-02T12:34:12,464 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1696249996848.737b8e61c96d872ea2ba4221e6c477e8. 2023-10-02T12:34:12,465 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing df86f7708dbdd71bc4105e1e70256952, disabling compactions & flushes 2023-10-02T12:34:12,465 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1696249976062.df86f7708dbdd71bc4105e1e70256952. 2023-10-02T12:34:12,465 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1696249976062.df86f7708dbdd71bc4105e1e70256952. 2023-10-02T12:34:12,465 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1696249976062.df86f7708dbdd71bc4105e1e70256952. after waiting 0 ms 2023-10-02T12:34:12,465 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1696249976062.df86f7708dbdd71bc4105e1e70256952. 2023-10-02T12:34:12,466 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing df86f7708dbdd71bc4105e1e70256952 1/1 column families, dataSize=249 B heapSize=1.02 KB 2023-10-02T12:34:12,482 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/hbase/namespace/df86f7708dbdd71bc4105e1e70256952/.tmp/info/5bb864691c0645d685e8f0d8281639f1 is 45, key is default/info:d/1696249977057/Put/seqid=0 2023-10-02T12:34:12,552 DEBUG [RS:0;jenkins-hbase3:38375 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 1d3e651667b9d2576aabb8501f40423e, 567521894cd008d1b0178b9ce1b98f67, ced4de67d56921f0b2e4ff571d6ad352, df86f7708dbdd71bc4105e1e70256952 2023-10-02T12:34:12,752 DEBUG [RS:0;jenkins-hbase3:38375 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 1d3e651667b9d2576aabb8501f40423e, 567521894cd008d1b0178b9ce1b98f67, ced4de67d56921f0b2e4ff571d6ad352, df86f7708dbdd71bc4105e1e70256952 2023-10-02T12:34:12,841 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=172 B at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/hbase/meta/1588230740/.tmp/rep_barrier/e6be557979eb479b87cc13a6a0802af4 2023-10-02T12:34:12,879 INFO [Socket Reader #1 for port 0 {}] ipc.Server$Connection(2001): Auth successful for appattempt_1696250022245_0001_000001 (auth:SIMPLE) 2023-10-02T12:34:12,879 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/hbase/meta/1588230740/.tmp/table/90ee0b2a985641eabc89baf4ed08a73d is 84, key is table1,,1696250006368.0daec27bd9d108cbf260f6b50d5177c8./table:/1696250010817/DeleteFamily/seqid=0 2023-10-02T12:34:12,887 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/hbase/namespace/df86f7708dbdd71bc4105e1e70256952/.tmp/info/5bb864691c0645d685e8f0d8281639f1 2023-10-02T12:34:12,890 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.22 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/hbase/meta/1588230740/.tmp/table/90ee0b2a985641eabc89baf4ed08a73d 2023-10-02T12:34:12,895 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/hbase/namespace/df86f7708dbdd71bc4105e1e70256952/.tmp/info/5bb864691c0645d685e8f0d8281639f1 as hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/hbase/namespace/df86f7708dbdd71bc4105e1e70256952/info/5bb864691c0645d685e8f0d8281639f1 2023-10-02T12:34:12,897 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/hbase/meta/1588230740/.tmp/info/53cdc018356246a79278d9625c3af3ea as hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/hbase/meta/1588230740/info/53cdc018356246a79278d9625c3af3ea 2023-10-02T12:34:12,901 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/hbase/namespace/df86f7708dbdd71bc4105e1e70256952/info/5bb864691c0645d685e8f0d8281639f1, entries=7, sequenceid=11, filesize=5.1 K 2023-10-02T12:34:12,902 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~249 B/249, heapSize ~1.01 KB/1032, currentSize=0 B/0 for df86f7708dbdd71bc4105e1e70256952 in 437ms, sequenceid=11, compaction requested=false 2023-10-02T12:34:12,902 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-10-02T12:34:12,903 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/hbase/meta/1588230740/info/53cdc018356246a79278d9625c3af3ea, entries=74, sequenceid=63, filesize=14.4 K 2023-10-02T12:34:12,904 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/hbase/meta/1588230740/.tmp/rep_barrier/e6be557979eb479b87cc13a6a0802af4 as hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/hbase/meta/1588230740/rep_barrier/e6be557979eb479b87cc13a6a0802af4 2023-10-02T12:34:12,917 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/hbase/meta/1588230740/rep_barrier/e6be557979eb479b87cc13a6a0802af4, entries=2, sequenceid=63, filesize=5.4 K 2023-10-02T12:34:12,930 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/hbase/meta/1588230740/.tmp/table/90ee0b2a985641eabc89baf4ed08a73d as hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/hbase/meta/1588230740/table/90ee0b2a985641eabc89baf4ed08a73d 2023-10-02T12:34:12,931 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/hbase/namespace/df86f7708dbdd71bc4105e1e70256952/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2023-10-02T12:34:12,934 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:34:12,937 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1696249976062.df86f7708dbdd71bc4105e1e70256952. 2023-10-02T12:34:12,937 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for df86f7708dbdd71bc4105e1e70256952: 2023-10-02T12:34:12,937 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1696249976062.df86f7708dbdd71bc4105e1e70256952. 2023-10-02T12:34:12,939 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 1d3e651667b9d2576aabb8501f40423e, disabling compactions & flushes 2023-10-02T12:34:12,939 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns3:test-16962499878512,,1696249991854.1d3e651667b9d2576aabb8501f40423e. 2023-10-02T12:34:12,939 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16962499878512,,1696249991854.1d3e651667b9d2576aabb8501f40423e. 2023-10-02T12:34:12,939 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16962499878512,,1696249991854.1d3e651667b9d2576aabb8501f40423e. after waiting 0 ms 2023-10-02T12:34:12,939 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16962499878512,,1696249991854.1d3e651667b9d2576aabb8501f40423e. 2023-10-02T12:34:12,948 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/hbase/meta/1588230740/table/90ee0b2a985641eabc89baf4ed08a73d, entries=17, sequenceid=63, filesize=6.0 K 2023-10-02T12:34:12,952 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/ns3/test-16962499878512/1d3e651667b9d2576aabb8501f40423e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-10-02T12:34:12,952 INFO [RS:0;jenkins-hbase3:38375 {}] regionserver.HRegionServer(1579): Waiting on 4 regions to close 2023-10-02T12:34:12,952 DEBUG [RS:0;jenkins-hbase3:38375 {}] regionserver.HRegionServer(1583): Online Regions={1588230740=hbase:meta,,1.1588230740, 1d3e651667b9d2576aabb8501f40423e=ns3:test-16962499878512,,1696249991854.1d3e651667b9d2576aabb8501f40423e., 567521894cd008d1b0178b9ce1b98f67=ns2:test-16962499878511,,1696249990628.567521894cd008d1b0178b9ce1b98f67., ced4de67d56921f0b2e4ff571d6ad352=ns4:test-16962499878513,,1696249992983.ced4de67d56921f0b2e4ff571d6ad352.} 2023-10-02T12:34:12,952 DEBUG [RS:0;jenkins-hbase3:38375 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 1d3e651667b9d2576aabb8501f40423e, 567521894cd008d1b0178b9ce1b98f67, ced4de67d56921f0b2e4ff571d6ad352 2023-10-02T12:34:12,953 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:34:12,955 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns3:test-16962499878512,,1696249991854.1d3e651667b9d2576aabb8501f40423e. 2023-10-02T12:34:12,955 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 1d3e651667b9d2576aabb8501f40423e: 2023-10-02T12:34:12,955 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-16962499878512,,1696249991854.1d3e651667b9d2576aabb8501f40423e. 2023-10-02T12:34:12,957 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 567521894cd008d1b0178b9ce1b98f67, disabling compactions & flushes 2023-10-02T12:34:12,957 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns2:test-16962499878511,,1696249990628.567521894cd008d1b0178b9ce1b98f67. 2023-10-02T12:34:12,957 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16962499878511,,1696249990628.567521894cd008d1b0178b9ce1b98f67. 2023-10-02T12:34:12,957 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16962499878511,,1696249990628.567521894cd008d1b0178b9ce1b98f67. after waiting 0 ms 2023-10-02T12:34:12,959 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~13.51 KB/13834, heapSize ~23.95 KB/24528, currentSize=0 B/0 for 1588230740 in 1008ms, sequenceid=63, compaction requested=false 2023-10-02T12:34:12,959 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-10-02T12:34:12,963 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16962499878511,,1696249990628.567521894cd008d1b0178b9ce1b98f67. 2023-10-02T12:34:12,963 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 567521894cd008d1b0178b9ce1b98f67 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-10-02T12:34:12,981 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/hbase/meta/1588230740/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=1 2023-10-02T12:34:12,982 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:34:12,982 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-10-02T12:34:12,983 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-10-02T12:34:12,983 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-10-02T12:34:12,984 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-10-02T12:34:12,989 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/ns2/test-16962499878511/567521894cd008d1b0178b9ce1b98f67/.tmp/f/174c1de7e1a14d258078448cd4b46508 is 37, key is row10/f:q1/1696249991760/Put/seqid=0 2023-10-02T12:34:12,993 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/ns2/test-16962499878511/567521894cd008d1b0178b9ce1b98f67/.tmp/f/174c1de7e1a14d258078448cd4b46508 2023-10-02T12:34:13,000 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/ns2/test-16962499878511/567521894cd008d1b0178b9ce1b98f67/.tmp/f/174c1de7e1a14d258078448cd4b46508 as hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/ns2/test-16962499878511/567521894cd008d1b0178b9ce1b98f67/f/174c1de7e1a14d258078448cd4b46508 2023-10-02T12:34:13,008 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/ns2/test-16962499878511/567521894cd008d1b0178b9ce1b98f67/f/174c1de7e1a14d258078448cd4b46508, entries=99, sequenceid=103, filesize=8.2 K 2023-10-02T12:34:13,009 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 567521894cd008d1b0178b9ce1b98f67 in 46ms, sequenceid=103, compaction requested=false 2023-10-02T12:34:13,029 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/ns2/test-16962499878511/567521894cd008d1b0178b9ce1b98f67/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-10-02T12:34:13,030 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:34:13,031 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns2:test-16962499878511,,1696249990628.567521894cd008d1b0178b9ce1b98f67. 2023-10-02T12:34:13,031 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 567521894cd008d1b0178b9ce1b98f67: 2023-10-02T12:34:13,031 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-16962499878511,,1696249990628.567521894cd008d1b0178b9ce1b98f67. 2023-10-02T12:34:13,033 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing ced4de67d56921f0b2e4ff571d6ad352, disabling compactions & flushes 2023-10-02T12:34:13,033 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns4:test-16962499878513,,1696249992983.ced4de67d56921f0b2e4ff571d6ad352. 2023-10-02T12:34:13,033 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16962499878513,,1696249992983.ced4de67d56921f0b2e4ff571d6ad352. 2023-10-02T12:34:13,033 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16962499878513,,1696249992983.ced4de67d56921f0b2e4ff571d6ad352. after waiting 0 ms 2023-10-02T12:34:13,033 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16962499878513,,1696249992983.ced4de67d56921f0b2e4ff571d6ad352. 2023-10-02T12:34:13,037 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/data/ns4/test-16962499878513/ced4de67d56921f0b2e4ff571d6ad352/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-10-02T12:34:13,038 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:34:13,039 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns4:test-16962499878513,,1696249992983.ced4de67d56921f0b2e4ff571d6ad352. 2023-10-02T12:34:13,039 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for ced4de67d56921f0b2e4ff571d6ad352: 2023-10-02T12:34:13,039 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-16962499878513,,1696249992983.ced4de67d56921f0b2e4ff571d6ad352. 2023-10-02T12:34:13,153 INFO [RS:0;jenkins-hbase3:38375 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,38375,1696249972520; all regions closed. 2023-10-02T12:34:13,163 DEBUG [RS:0;jenkins-hbase3:38375 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/oldWALs 2023-10-02T12:34:13,163 INFO [RS:0;jenkins-hbase3:38375 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C38375%2C1696249972520.meta:.meta(num 1696250013204) 2023-10-02T12:34:13,171 DEBUG [RS:0;jenkins-hbase3:38375 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/oldWALs 2023-10-02T12:34:13,171 INFO [RS:0;jenkins-hbase3:38375 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C38375%2C1696249972520:(num 1696250013176) 2023-10-02T12:34:13,171 DEBUG [RS:0;jenkins-hbase3:38375 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:13,171 INFO [RS:0;jenkins-hbase3:38375 {}] regionserver.LeaseManager(133): Closed leases 2023-10-02T12:34:13,171 INFO [RS:0;jenkins-hbase3:38375 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2023-10-02T12:34:13,172 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-10-02T12:34:13,174 INFO [RS:0;jenkins-hbase3:38375 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:38375 2023-10-02T12:34:13,181 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-10-02T12:34:13,181 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rs/jenkins-hbase3.apache.org,38375,1696249972520 2023-10-02T12:34:13,190 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,38375,1696249972520] 2023-10-02T12:34:13,190 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,38375,1696249972520; numProcessing=1 2023-10-02T12:34:13,192 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(224): Node /1/draining/jenkins-hbase3.apache.org,38375,1696249972520 already deleted, retry=false 2023-10-02T12:34:13,192 INFO [RegionServerTracker-0 {}] master.ServerManager(569): Cluster shutdown set; jenkins-hbase3.apache.org,38375,1696249972520 expired; onlineServers=0 2023-10-02T12:34:13,192 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,45341,1696249969200' ***** 2023-10-02T12:34:13,192 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2531): STOPPED: Cluster shutdown set; onlineServer=0 2023-10-02T12:34:13,194 DEBUG [M:0;jenkins-hbase3:45341 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb9d6f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-10-02T12:34:13,194 INFO [M:0;jenkins-hbase3:45341 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-10-02T12:34:13,197 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/master 2023-10-02T12:34:13,197 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:34:13,197 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-10-02T12:34:13,242 INFO [M:0;jenkins-hbase3:45341 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@11de04f5{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-10-02T12:34:13,243 INFO [M:0;jenkins-hbase3:45341 {}] server.AbstractConnector(383): Stopped ServerConnector@3c70a669{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-10-02T12:34:13,243 INFO [M:0;jenkins-hbase3:45341 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-10-02T12:34:13,244 INFO [M:0;jenkins-hbase3:45341 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@5eae4c46{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-10-02T12:34:13,246 INFO [M:0;jenkins-hbase3:45341 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@7d8f62fb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/hadoop.log.dir/,STOPPED} 2023-10-02T12:34:13,250 INFO [M:0;jenkins-hbase3:45341 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,45341,1696249969200 2023-10-02T12:34:13,250 INFO [M:0;jenkins-hbase3:45341 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,45341,1696249969200; all regions closed. 2023-10-02T12:34:13,250 DEBUG [M:0;jenkins-hbase3:45341 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:13,250 INFO [M:0;jenkins-hbase3:45341 {}] master.HMaster(1665): Stopping master jetty server 2023-10-02T12:34:13,251 INFO [M:0;jenkins-hbase3:45341 {}] server.AbstractConnector(383): Stopped ServerConnector@eaa0ccc{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-10-02T12:34:13,252 DEBUG [M:0;jenkins-hbase3:45341 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-10-02T12:34:13,252 INFO [M:0;jenkins-hbase3:45341 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-10-02T12:34:13,252 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-10-02T12:34:13,252 DEBUG [M:0;jenkins-hbase3:45341 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x22751e54 to 127.0.0.1:49524 2023-10-02T12:34:13,252 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1696249974846 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1696249974846,5,FailOnTimeoutGroup] 2023-10-02T12:34:13,252 DEBUG [M:0;jenkins-hbase3:45341 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:13,252 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1696249974845 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1696249974845,5,FailOnTimeoutGroup] 2023-10-02T12:34:13,252 DEBUG [M:0;jenkins-hbase3:45341 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-10-02T12:34:13,253 INFO [M:0;jenkins-hbase3:45341 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-10-02T12:34:13,253 DEBUG [M:0;jenkins-hbase3:45341 {}] master.HMaster(1688): Stopping service threads 2023-10-02T12:34:13,253 INFO [M:0;jenkins-hbase3:45341 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-10-02T12:34:13,253 ERROR [M:0;jenkins-hbase3:45341 {}] procedure2.ProcedureExecutor(654): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-2,5,PEWorkerGroup] 2023-10-02T12:34:13,254 INFO [M:0;jenkins-hbase3:45341 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-10-02T12:34:13,254 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-10-02T12:34:13,254 DEBUG [M:0;jenkins-hbase3:45341 {}] zookeeper.ZKUtil(347): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Unable to get data of znode /1/master because node does not exist (not an error) 2023-10-02T12:34:13,254 WARN [M:0;jenkins-hbase3:45341 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-10-02T12:34:13,254 INFO [M:0;jenkins-hbase3:45341 {}] assignment.AssignmentManager(382): Stopping assignment manager 2023-10-02T12:34:13,255 INFO [M:0;jenkins-hbase3:45341 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-10-02T12:34:13,257 DEBUG [M:0;jenkins-hbase3:45341 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-10-02T12:34:13,274 INFO [M:0;jenkins-hbase3:45341 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:34:13,274 DEBUG [M:0;jenkins-hbase3:45341 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:34:13,274 DEBUG [M:0;jenkins-hbase3:45341 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-10-02T12:34:13,274 DEBUG [M:0;jenkins-hbase3:45341 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:34:13,274 INFO [M:0;jenkins-hbase3:45341 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=203.16 KB heapSize=244.13 KB 2023-10-02T12:34:13,291 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-10-02T12:34:13,291 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38375-0x1017329d6f10001, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-10-02T12:34:13,291 INFO [RS:0;jenkins-hbase3:38375 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,38375,1696249972520; zookeeper connection closed. 2023-10-02T12:34:13,291 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@660017e8 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@660017e8 2023-10-02T12:34:13,291 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-10-02T12:34:13,315 DEBUG [M:0;jenkins-hbase3:45341 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ee1e7a6715c94d298fcfa43050d6f2cb is 88, key is hbase:meta,,1/info:sn/1696249975347/Put/seqid=0 2023-10-02T12:34:13,330 INFO [M:0;jenkins-hbase3:45341 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ee1e7a6715c94d298fcfa43050d6f2cb 2023-10-02T12:34:13,349 DEBUG [M:0;jenkins-hbase3:45341 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1ff4240d570a4fa8ae9c080197984f4c is 988, key is \x00\x00\x00\x00\x00\x00\x00\x1D/proc:d/1696249997694/Put/seqid=0 2023-10-02T12:34:13,354 INFO [M:0;jenkins-hbase3:45341 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=202.56 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1ff4240d570a4fa8ae9c080197984f4c 2023-10-02T12:34:13,359 INFO [M:0;jenkins-hbase3:45341 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1ff4240d570a4fa8ae9c080197984f4c 2023-10-02T12:34:13,386 DEBUG [M:0;jenkins-hbase3:45341 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/af33a23854714d0f9fb3ab626c007e3f is 82, key is jenkins-hbase3.apache.org,38375,1696249972520/rs:state/1696249974976/Put/seqid=0 2023-10-02T12:34:13,392 INFO [M:0;jenkins-hbase3:45341 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/af33a23854714d0f9fb3ab626c007e3f 2023-10-02T12:34:13,399 DEBUG [M:0;jenkins-hbase3:45341 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ee1e7a6715c94d298fcfa43050d6f2cb as hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ee1e7a6715c94d298fcfa43050d6f2cb 2023-10-02T12:34:13,405 INFO [M:0;jenkins-hbase3:45341 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ee1e7a6715c94d298fcfa43050d6f2cb, entries=8, sequenceid=480, filesize=5.5 K 2023-10-02T12:34:13,406 DEBUG [M:0;jenkins-hbase3:45341 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1ff4240d570a4fa8ae9c080197984f4c as hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1ff4240d570a4fa8ae9c080197984f4c 2023-10-02T12:34:13,413 INFO [M:0;jenkins-hbase3:45341 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1ff4240d570a4fa8ae9c080197984f4c 2023-10-02T12:34:13,413 INFO [M:0;jenkins-hbase3:45341 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1ff4240d570a4fa8ae9c080197984f4c, entries=53, sequenceid=480, filesize=16.7 K 2023-10-02T12:34:13,414 DEBUG [M:0;jenkins-hbase3:45341 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/af33a23854714d0f9fb3ab626c007e3f as hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/af33a23854714d0f9fb3ab626c007e3f 2023-10-02T12:34:13,420 INFO [M:0;jenkins-hbase3:45341 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45521/user/jenkins/test-data/38c796ab-7db6-c3bd-e7f9-85df82522560/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/af33a23854714d0f9fb3ab626c007e3f, entries=1, sequenceid=480, filesize=5.1 K 2023-10-02T12:34:13,422 INFO [M:0;jenkins-hbase3:45341 {}] regionserver.HRegion(3022): Finished flush of dataSize ~203.16 KB/208033, heapSize ~243.83 KB/249680, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 147ms, sequenceid=480, compaction requested=false 2023-10-02T12:34:13,444 INFO [M:0;jenkins-hbase3:45341 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:34:13,444 DEBUG [M:0;jenkins-hbase3:45341 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-10-02T12:34:13,469 INFO [M:0;jenkins-hbase3:45341 {}] flush.MasterFlushTableProcedureManager(85): stop: server shutting down. 2023-10-02T12:34:13,469 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-10-02T12:34:13,469 INFO [M:0;jenkins-hbase3:45341 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-10-02T12:34:13,470 INFO [M:0;jenkins-hbase3:45341 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:45341 2023-10-02T12:34:13,473 DEBUG [M:0;jenkins-hbase3:45341 {}] zookeeper.RecoverableZooKeeper(224): Node /1/rs/jenkins-hbase3.apache.org,45341,1696249969200 already deleted, retry=false 2023-10-02T12:34:13,577 INFO [M:0;jenkins-hbase3:45341 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,45341,1696249969200; zookeeper connection closed. 2023-10-02T12:34:13,577 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-10-02T12:34:13,577 DEBUG [Listener at localhost/35185-EventThread {}] zookeeper.ZKWatcher(604): master:45341-0x1017329d6f10000, quorum=127.0.0.1:49524, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-10-02T12:34:13,620 WARN [Listener at jenkins-hbase3.apache.org/43893 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-10-02T12:34:13,673 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-10-02T12:34:13,756 INFO [Finalizer {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-02T12:34:13,756 DEBUG [Finalizer {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5691222f to 127.0.0.1:49524 2023-10-02T12:34:13,756 DEBUG [Finalizer {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:13,852 WARN [BP-2107663488-172.31.12.81-1696249963873 heartbeating to localhost/127.0.0.1:45521 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-10-02T12:34:13,852 WARN [BP-2107663488-172.31.12.81-1696249963873 heartbeating to localhost/127.0.0.1:45521 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-2107663488-172.31.12.81-1696249963873 (Datanode Uuid 9eb68d63-ba42-497c-97b5-303b9b9c3b18) service to localhost/127.0.0.1:45521 2023-10-02T12:34:13,854 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/cluster_f3ba57df-0a4a-7a4e-2411-dac870a6ebff/dfs/data/data1/current/BP-2107663488-172.31.12.81-1696249963873 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-10-02T12:34:13,854 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/3a04bcc6-6bb4-3fd3-6bda-affe3961b31a/cluster_f3ba57df-0a4a-7a4e-2411-dac870a6ebff/dfs/data/data2/current/BP-2107663488-172.31.12.81-1696249963873 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-10-02T12:34:13,944 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-10-02T12:34:14,070 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2023-10-02T12:34:14,097 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-10-02T12:34:14,097 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] hbase.HBaseTestingUtility(2857): Stopping mini mapreduce cluster... 2023-10-02T12:34:14,110 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-10-02T12:34:14,240 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-10-02T12:34:14,243 WARN [1654734328@qtp-1942406187-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:34597 {}] http.HttpServer2$SelectChannelConnectorWithSafeStartup(546): HttpServer Acceptor: isRunning is false. Rechecking. 2023-10-02T12:34:14,243 WARN [1654734328@qtp-1942406187-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:34597 {}] http.HttpServer2$SelectChannelConnectorWithSafeStartup(555): HttpServer Acceptor: isRunning is false 2023-10-02T12:34:14,352 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:34:14,502 ERROR [Thread[Thread-492,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-10-02T12:34:14,502 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-10-02T12:34:14,609 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2023-10-02T12:34:14,616 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(61): Returning, interrupted : java.lang.InterruptedException 2023-10-02T12:34:14,620 ERROR [Thread[Thread-502,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-10-02T12:34:14,632 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-10-02T12:34:14,740 ERROR [Thread[Thread-467,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-10-02T12:34:14,741 INFO [Listener at jenkins-hbase3.apache.org/43893 {}] hbase.HBaseTestingUtility(2860): Mini mapreduce cluster stopped